Protocol Buffers

It is known that Cosmos SDK uses protocol buffers extensively, this docuemnt is meant to provide a guide on how it is used in the cosmos-sdk.

It is known that Cosmos SDK uses protocol buffers extensively, this docuemnt is meant to provide a guide on how it is used in the cosmos-sdk.

To generate the proto file, the Cosmos SDK uses a docker image, this image is provided to all to use as well. The latest version is ghcr.io/cosmos/proto-builder:0.12.x

Below is the example of the Cosmos SDK’s commands for generating, linting, and formatting protobuf files that can be reused in any applications makefile.

#!/usr/bin/make -f

PACKAGES_NOSIMULATION=$(shell go list ./... | grep -v '/simulation')

PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation')

export VERSION := $(shell echo $(shell git describe --always --match "v*") | sed 's/^v/')

export TMVERSION := $(shell go list -m github.com/tendermint/tendermint | sed 's:.* ::')

export COMMIT := $(shell git log -1 --format='%H')

LEDGER_ENABLED ?= true
BINDIR ?= $(GOPATH)/bin
BUILDDIR ?= $(CURDIR)/build
SIMAPP = ./simapp
MOCKS_DIR = $(CURDIR)/tests/mocks
HTTPS_GIT := https://github.com/cosmos/cosmos-sdk.git
DOCKER := $(shell which docker)

PROJECT_NAME = $(shell git remote get-url origin | xargs basename -s .git)

DOCS_DOMAIN=docs.cosmos.network
# RocksDB is a native dependency, so we don't assume the library is installed.
# Instead, it must be explicitly enabled and we warn when it is not.
ENABLE_ROCKSDB ?= false

# process build tags
build_tags = netgo
    ifeq ($(LEDGER_ENABLED),true)
    ifeq ($(OS),Windows_NT)

GCCEXE = $(shell where gcc.exe 2> NUL)
    ifeq ($(GCCEXE),)
      $(error gcc.exe not installed for ledger support, please install or set LEDGER_ENABLED=false)

else
      build_tags += ledger
    endif
  else
    UNAME_S = $(shell uname -s)
    ifeq ($(UNAME_S),OpenBSD)
      $(warning OpenBSD detected, disabling ledger support (https://github.com/cosmos/cosmos-sdk/issues/1988))

else
      GCC = $(shell command -v gcc 2> /dev/null)
    ifeq ($(GCC),)
        $(error gcc not installed for ledger support, please install or set LEDGER_ENABLED=false)

else
        build_tags += ledger
      endif
    endif
  endif
endif
    ifeq (secp,$(findstring secp,$(COSMOS_BUILD_OPTIONS)))

build_tags += libsecp256k1_sdk
endif
    ifeq (legacy,$(findstring legacy,$(COSMOS_BUILD_OPTIONS)))

build_tags += app_v1
endif
    whitespace :=
whitespace += $(whitespace)
    comma := ,
build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags))

# process linker flags

ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=sim \
		  -X github.com/cosmos/cosmos-sdk/version.AppName=simd \
		  -X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \
		  -X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
		  -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)" \
			-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(TMVERSION)
    ifeq ($(ENABLE_ROCKSDB),true)

BUILD_TAGS += rocksdb_build
  test_tags += rocksdb_build
endif

# DB backend selection
    ifeq (cleveldb,$(findstring cleveldb,$(COSMOS_BUILD_OPTIONS)))

build_tags += gcc
endif
    ifeq (badgerdb,$(findstring badgerdb,$(COSMOS_BUILD_OPTIONS)))

BUILD_TAGS += badgerdb
endif
# handle rocksdb
    ifeq (rocksdb,$(findstring rocksdb,$(COSMOS_BUILD_OPTIONS)))
    ifneq ($(ENABLE_ROCKSDB),true)
    $(error Cannot use RocksDB backend unless ENABLE_ROCKSDB=true)

endif
  CGO_ENABLED=1
  BUILD_TAGS += rocksdb
endif
# handle boltdb
    ifeq (boltdb,$(findstring boltdb,$(COSMOS_BUILD_OPTIONS)))

BUILD_TAGS += boltdb
endif
    ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS)))

ldflags += -w -s
endif
ldflags += $(LDFLAGS)
    ldflags := $(strip $(ldflags))

build_tags += $(BUILD_TAGS)

build_tags := $(strip $(build_tags))

BUILD_FLAGS := -tags "$(build_tags)" -ldflags '$(ldflags)'
# check for nostrip option
    ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS)))

BUILD_FLAGS += -trimpath
endif

# Check for debug option
    ifeq (debug,$(findstring debug,$(COSMOS_BUILD_OPTIONS)))

BUILD_FLAGS += -gcflags "all=-N -l"
endif

all: tools build lint test vulncheck

# The below include contains the tools and runsim targets.
include contrib/devtools/Makefile

###############################################################################
###                                  Build                                  ###
###############################################################################

BUILD_TARGETS := build install

build: BUILD_ARGS=-o $(BUILDDIR)/

build-linux-amd64:
	GOOS=linux GOARCH=amd64 LEDGER_ENABLED=false $(MAKE)

build
	
build-linux-arm64:
	GOOS=linux GOARCH=arm64 LEDGER_ENABLED=false $(MAKE)

build

$(BUILD_TARGETS): go.sum $(BUILDDIR)/
	cd ${
    CURRENT_DIR
}/simapp && go $@ -mod=readonly $(BUILD_FLAGS) $(BUILD_ARGS) ./...

$(BUILDDIR)/:
	mkdir -p $(BUILDDIR)/

cosmovisor:
	$(MAKE) -C tools/cosmovisor cosmovisor

.PHONY: build build-linux-amd64 build-linux-arm64 cosmovisor

mocks: $(MOCKS_DIR)
	@go install github.com/golang/mock/mockgen@v1.6.0
	sh ./scripts/mockgen.sh
.PHONY: mocks

vulncheck: $(BUILDDIR)/
	GOBIN=$(BUILDDIR)

go install golang.org/x/vuln/cmd/govulncheck@latest
	$(BUILDDIR)/govulncheck ./...

$(MOCKS_DIR):
	mkdir -p $(MOCKS_DIR)

distclean: clean tools-clean
clean:
	rm -rf \
    $(BUILDDIR)/ \
    artifacts/ \
    tmp-swagger-gen/

.PHONY: distclean clean

###############################################################################
###                          Tools & Dependencies                           ###
###############################################################################

go.sum: go.mod
	echo "Ensure dependencies have not been modified ..." >&2
	go mod verify
	go mod tidy

###############################################################################
###                              Documentation                              ###
###############################################################################

update-swagger-docs: statik
	$(BINDIR)/statik -src=client/docs/swagger-ui -dest=client/docs -f -m
	@if [ -n "$(git status --porcelain)" ]; then \
        echo "\033[91mSwagger docs are out of sync!!!\033[0m";\
        exit 1;\
    else \
        echo "\033[92mSwagger docs are in sync\033[0m";\
    fi
.PHONY: update-swagger-docs

godocs:
	@echo "--> Wait a few seconds and visit http://localhost:6060/pkg/github.com/cosmos/cosmos-sdk/types"
	godoc -http=:6060

# This builds the docs.cosmos.network docs using docusaurus.
# Old documentation, which have not been migrated to docusaurus are generated with vuepress.
build-docs:
	@echo "building docusaurus docs"
	@cd docs && npm ci && npm run build
	mv docs/build ~/output

	@echo "building old docs"
	@cd docs && \
			while read -r branch path_prefix; do \
			echo "building vuepress ${
    branch
}

docs" ; \
			(git clean -fdx && git reset --hard && git checkout ${
    branch
} && npm install && VUEPRESS_BASE="/${
    path_prefix
}/" npm run build) ; \
			mkdir -p ~/output/${
    path_prefix
} ; \
			cp -r .vuepress/dist/* ~/output/${
    path_prefix
}/ ; \
	done < vuepress_versions ;	

	@echo "setup domain"
	@echo $(DOCS_DOMAIN) > ~/output/CNAME

.PHONY: build-docs

###############################################################################
###                           Tests & Simulation                            ###
###############################################################################

test: test-unit
test-e2e:
	$(MAKE) -C tests test-e2e
test-e2e-cov:
	$(MAKE) -C tests test-e2e-cov
test-integration:
	$(MAKE) -C tests test-integration
test-integration-cov:
	$(MAKE) -C tests test-integration-cov
test-all: test-unit test-e2e test-integration test-ledger-mock test-race

TEST_PACKAGES=./...
TEST_TARGETS := test-unit test-unit-amino test-unit-proto test-ledger-mock test-race test-ledger test-race

# Test runs-specific rules. To add a new test target, just add
# a new rule, customise ARGS or TEST_PACKAGES ad libitum, and
# append the new rule to the TEST_TARGETS list.
test-unit: test_tags += cgo ledger test_ledger_mock norace
test-unit-amino: test_tags += ledger test_ledger_mock test_amino norace
test-ledger: test_tags += cgo ledger norace
test-ledger-mock: test_tags += ledger test_ledger_mock norace
test-race: test_tags += cgo ledger test_ledger_mock
test-race: ARGS=-race
test-race: TEST_PACKAGES=$(PACKAGES_NOSIMULATION)
$(TEST_TARGETS): run-tests

# check-* compiles and collects tests without running them
# note: go test -c doesn't support multiple packages yet (https://github.com/golang/go/issues/15513)

CHECK_TEST_TARGETS := check-test-unit check-test-unit-amino
check-test-unit: test_tags += cgo ledger test_ledger_mock norace
check-test-unit-amino: test_tags += ledger test_ledger_mock test_amino norace
$(CHECK_TEST_TARGETS): EXTRA_ARGS=-run=none
$(CHECK_TEST_TARGETS): run-tests

ARGS += -tags "$(test_tags)"
SUB_MODULES = $(shell find . -type f -name 'go.mod' -print0 | xargs -0 -n1 dirname | sort)

CURRENT_DIR = $(shell pwd)

run-tests:
    ifneq (,$(shell which tparse 2>/dev/null))
	@echo "Starting unit tests"; \
	finalec=0; \
    for module in $(SUB_MODULES); do \
		cd ${
    CURRENT_DIR
}/$module; \
		echo "Running unit tests for module $module"; \
		go test -mod=readonly -json $(ARGS) $(TEST_PACKAGES) ./... | tparse; \
		ec=$?; \
    if [ "$ec" -ne '0' ]; then finalec=$ec; fi; \
	done; \
	exit $finalec
else
	@echo "Starting unit tests"; \
	finalec=0; \
    for module in $(SUB_MODULES); do \
		cd ${
    CURRENT_DIR
}/$module; \
		echo "Running unit tests for module $module"; \
		go test -mod=readonly $(ARGS) $(TEST_PACKAGES) ./... ; \
		ec=$?; \
    if [ "$ec" -ne '0' ]; then finalec=$ec; fi; \
	done; \
	exit $finalec
endif

.PHONY: run-tests test test-all $(TEST_TARGETS)

test-sim-nondeterminism:
	@echo "Running non-determinism test..."
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
		-NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h

test-sim-custom-genesis-fast:
	@echo "Running custom genesis simulation..."
	@echo "By default, ${
    HOME
}/.gaiad/config/genesis.json will be used."
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -run TestFullAppSimulation -Genesis=${
    HOME
}/.gaiad/config/genesis.json \
		-Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h

test-sim-import-export: runsim
	@echo "Running application import/export simulation. This may take several minutes..."
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppImportExport

test-sim-after-import: runsim
	@echo "Running application simulation-after-import. This may take several minutes..."
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppSimulationAfterImport

test-sim-custom-genesis-multi-seed: runsim
	@echo "Running multi-seed custom genesis simulation..."
	@echo "By default, ${
    HOME
}/.gaiad/config/genesis.json will be used."
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Genesis=${
    HOME
}/.gaiad/config/genesis.json -SimAppPkg=. -ExitOnFail 400 5 TestFullAppSimulation

test-sim-multi-seed-long: runsim
	@echo "Running long multi-seed application simulation. This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 500 50 TestFullAppSimulation

test-sim-multi-seed-short: runsim
	@echo "Running short multi-seed application simulation. This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 10 TestFullAppSimulation

test-sim-benchmark-invariants:
	@echo "Running simulation invariant benchmarks..."
	cd ${
    CURRENT_DIR
}/simapp && @go test -mod=readonly -benchmem -bench=BenchmarkInvariants -run=^$ \
	-Enabled=true -NumBlocks=1000 -BlockSize=200 \
	-Period=1 -Commit=true -Seed=57 -v -timeout 24h

.PHONY: \
test-sim-nondeterminism \
test-sim-custom-genesis-fast \
test-sim-import-export \
test-sim-after-import \
test-sim-custom-genesis-multi-seed \
test-sim-multi-seed-short \
test-sim-multi-seed-long \
test-sim-benchmark-invariants

SIM_NUM_BLOCKS ?= 500
SIM_BLOCK_SIZE ?= 200
SIM_COMMIT ?= true

test-sim-benchmark:
	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
	@go test -mod=readonly -benchmem -run=^$ $(SIMAPP) -bench ^BenchmarkFullAppSimulation$  \
		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h

test-sim-profile:
	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
	@go test -mod=readonly -benchmem -run=^$ $(SIMAPP) -bench ^BenchmarkFullAppSimulation$ \
		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out

.PHONY: test-sim-profile test-sim-benchmark

test-rosetta:
	docker build -t rosetta-ci:latest -f contrib/rosetta/rosetta-ci/Dockerfile .
	docker-compose -f contrib/rosetta/docker-compose.yaml up --abort-on-container-exit --exit-code-from test_rosetta --build
.PHONY: test-rosetta

benchmark:
	@go test -mod=readonly -bench=. $(PACKAGES_NOSIMULATION)
.PHONY: benchmark

###############################################################################
###                                Linting                                  ###
###############################################################################

golangci_lint_cmd=golangci-lint
golangci_version=v1.50.0

lint:
	@echo "--> Running linter"
	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(golangci_version)
	@$(golangci_lint_cmd)

run --timeout=10m

lint-fix:
	@echo "--> Running linter"
	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(golangci_version)
	@$(golangci_lint_cmd)

run --fix --out-format=tab --issues-exit-code=0

.PHONY: lint lint-fix
    format:
	@go install mvdan.cc/gofumpt@latest
	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(golangci_version)

find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/docs/statik/statik.go" -not -path "./tests/mocks/*" -not -name "*.pb.go" -not -name "*.pb.gw.go" -not -name "*.pulsar.go" -not -path "./crypto/keys/secp256k1/*" | xargs gofumpt -w -l
	$(golangci_lint_cmd)

run --fix
.PHONY: format

###############################################################################
###                                 Devdoc                                  ###
###############################################################################

DEVDOC_SAVE = docker commit `docker ps -a -n 1 -q` devdoc:local

devdoc-init:
	$(DOCKER)

run -it -v "$(CURDIR):/go/src/github.com/cosmos/cosmos-sdk" -w "/go/src/github.com/cosmos/cosmos-sdk" tendermint/devdoc echo
	# TODO make this safer
	$(call DEVDOC_SAVE)

devdoc:
	$(DOCKER)

run -it -v "$(CURDIR):/go/src/github.com/cosmos/cosmos-sdk" -w "/go/src/github.com/cosmos/cosmos-sdk" devdoc:local bash

devdoc-save:
	# TODO make this safer
	$(call DEVDOC_SAVE)

devdoc-clean:
	docker rmi -f $(docker images -f "dangling=true" -q)

devdoc-update:
	docker pull tendermint/devdoc

.PHONY: devdoc devdoc-clean devdoc-init devdoc-save devdoc-update

###############################################################################
###                                Protobuf                                 ###
###############################################################################

protoVer=0.11.2
protoImageName=ghcr.io/cosmos/proto-builder:$(protoVer)

protoImage=$(DOCKER)

run --rm -v $(CURDIR):/workspace --workdir /workspace $(protoImageName)

proto-all: proto-format proto-lint proto-gen

proto-gen:
	@echo "Generating Protobuf files"
	@$(protoImage)

sh ./scripts/protocgen.sh

proto-swagger-gen:
	@echo "Generating Protobuf Swagger"
	@$(protoImage)

sh ./scripts/protoc-swagger-gen.sh

proto-format:
	@$(protoImage)

find ./ -name "*.proto" -exec clang-format -i {
} \;

proto-lint:
	@$(protoImage)

buf lint --error-format=json

proto-check-breaking:
	@$(protoImage)

buf breaking --against $(HTTPS_GIT)#branch=main

TM_URL              = https://raw.githubusercontent.com/tendermint/tendermint/v0.37.0-rc2/proto/tendermint

TM_CRYPTO_TYPES     = proto/tendermint/crypto
TM_ABCI_TYPES       = proto/tendermint/abci
TM_TYPES            = proto/tendermint/types
TM_VERSION          = proto/tendermint/version
TM_LIBS             = proto/tendermint/libs/bits
TM_P2P              = proto/tendermint/p2p

proto-update-deps:
	@echo "Updating Protobuf dependencies"

	@mkdir -p $(TM_ABCI_TYPES)
	@curl -sSL $(TM_URL)/abci/types.proto > $(TM_ABCI_TYPES)/types.proto

	@mkdir -p $(TM_VERSION)
	@curl -sSL $(TM_URL)/version/types.proto > $(TM_VERSION)/types.proto

	@mkdir -p $(TM_TYPES)
	@curl -sSL $(TM_URL)/types/types.proto > $(TM_TYPES)/types.proto
	@curl -sSL $(TM_URL)/types/evidence.proto > $(TM_TYPES)/evidence.proto
	@curl -sSL $(TM_URL)/types/params.proto > $(TM_TYPES)/params.proto
	@curl -sSL $(TM_URL)/types/validator.proto > $(TM_TYPES)/validator.proto
	@curl -sSL $(TM_URL)/types/block.proto > $(TM_TYPES)/block.proto

	@mkdir -p $(TM_CRYPTO_TYPES)
	@curl -sSL $(TM_URL)/crypto/proof.proto > $(TM_CRYPTO_TYPES)/proof.proto
	@curl -sSL $(TM_URL)/crypto/keys.proto > $(TM_CRYPTO_TYPES)/keys.proto

	@mkdir -p $(TM_LIBS)
	@curl -sSL $(TM_URL)/libs/bits/types.proto > $(TM_LIBS)/types.proto

	@mkdir -p $(TM_P2P)
	@curl -sSL $(TM_URL)/p2p/types.proto > $(TM_P2P)/types.proto

	$(DOCKER)

run --rm -v $(CURDIR)/proto:/workspace --workdir /workspace $(protoImageName)

buf mod update

.PHONY: proto-all proto-gen proto-swagger-gen proto-format proto-lint proto-check-breaking proto-update-deps

###############################################################################
###                                Localnet                                 ###
###############################################################################

localnet-build-env:
	$(MAKE) -C contrib/images simd-env
localnet-build-dlv:
	$(MAKE) -C contrib/images simd-dlv

localnet-build-nodes:
	$(DOCKER)

run --rm -v $(CURDIR)/.testnets:/data cosmossdk/simd \
			  testnet init-files --v 4 -o /data --starting-ip-address 192.168.10.2 --keyring-backend=test
	docker-compose up -d

localnet-stop:
	docker-compose down

# localnet-start will run a 4-node testnet locally. The nodes are
# based off the docker images in: ./contrib/images/simd-env
localnet-start: localnet-stop localnet-build-env localnet-build-nodes

# localnet-debug will run a 4-node testnet locally in debug mode
# you can read more about the debug mode here: ./contrib/images/simd-dlv/README.md
localnet-debug: localnet-stop localnet-build-dlv localnet-build-nodes

.PHONY: localnet-start localnet-stop localnet-debug localnet-build-env localnet-build-dlv localnet-build-nodes

###############################################################################
###                                rosetta                                  ###
###############################################################################
# builds rosetta test data dir
rosetta-data:
	-docker container rm data_dir_build
	docker build -t rosetta-ci:latest -f contrib/rosetta/rosetta-ci/Dockerfile .
	docker run --name data_dir_build -t rosetta-ci:latest sh /rosetta/data.sh
	docker cp data_dir_build:/tmp/data.tar.gz "$(CURDIR)/contrib/rosetta/rosetta-ci/data.tar.gz"
	docker container rm data_dir_build
.PHONY: rosetta-data

The script used to generate the protobuf files can be found in the scripts/ directory.

#!/usr/bin/env bash

# How to run manually:
# docker build --pull --rm -f "contrib/devtools/Dockerfile" -t cosmossdk-proto:latest "contrib/devtools"
# docker run --rm -v $(pwd):/workspace --workdir /workspace cosmossdk-proto sh ./scripts/protocgen.sh

set -e

echo "Generating gogo proto code"
cd proto
proto_dirs=$(find ./cosmos ./amino -path -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq)
for dir in $proto_dirs; do
  for file in $(find "${dir}" -maxdepth 1 -name '*.proto'); do
    # this regex checks if a proto file has its go_package set to cosmossdk.io/api/...
    # gogo proto files SHOULD ONLY be generated if this is false
    # we don't want gogo proto to run for proto files which are natively built for google.golang.org/protobuf
    if grep -q "option go_package" "$file" && grep -H -o -c 'option go_package.*cosmossdk.io/api' "$file" | grep -q ':0

Buf

Buf is a protobuf tool that abstracts the needs to use the complicated protoc toolchain on top of various other things that ensure you are using protobuf in accordance with the majority of the ecosystem. Within the cosmos-sdk repository there are a few files that have a buf prefix. Lets start with the top level and then dive into the various directories.

Workspace

At the root level directory a workspace is defined using buf workspaces. This helps if there are one or more protobuf containing directories in your project.

Cosmos SDK example:

version: v1
directories:
  - proto

Proto Directory

Next is the proto/ directory where all of our protobuf files live. In here there are many different buf files defined each serving a different purpose.

├── 05-depinject.md
├── buf.gen.gogo.yaml
├── buf.gen.pulsar.yaml
├── buf.gen.swagger.yaml
├── buf.lock
├── buf.md
├── buf.yaml
├── cosmos
└── tendermint

The above diagram all the files and directories within the Cosmos SDK proto/ directory.

buf.gen.gogo.yaml

buf.gen.gogo.yaml defines how the protobuf files should be generated for use with in the module. This file uses gogoproto, a separate generator from the google go-proto generator that makes working with various objects more ergonomic, and it has more performant encode and decode steps

version: v1
plugins:
  - name: gocosmos
    out: ..
    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/gogoproto/types/any
  - name: grpc-gateway
    out: ..
    opt: logtostderr=true,allow_colon_final_segments=true

Example of how to define gen files can be found here

buf.gen.pulsar.yaml

buf.gen.pulsar.yaml defines how protobuf files should be generated using the new golang apiv2 of protobuf. This generator is used instead of the google go-proto generator because it has some extra helpers for Cosmos SDK applications and will have more performant encode and decode than the google go-proto generator. You can follow the development of this generator here.

version: v1
managed:
  enabled: true
  go_package_prefix:
    default: cosmossdk.io/api
    except:
      - buf.build/googleapis/googleapis
      - buf.build/cosmos/gogo-proto
      - buf.build/cosmos/cosmos-proto
    override:
plugins:
  - name: go-pulsar
    out: ../api
    opt: paths=source_relative
  - name: go-grpc
    out: ../api
    opt: paths=source_relative

Example of how to define gen files can be found here

buf.gen.swagger.yaml

buf.gen.swagger.yaml generates the swagger documentation for the query and messages of the chain. This will only define the REST API end points that were defined in the query and msg servers. You can find examples of this here

version: v1
plugins:
  - name: swagger
    out: ../tmp-swagger-gen
    opt: logtostderr=true,fqn_for_swagger_name=true,simple_operation_ids=true

Example of how to define gen files can be found here

buf.lock

This is a autogenerated file based off the dependencies required by the .gen files. There is no need to copy the current one. If you depend on cosmos-sdk proto definitions a new entry for the Cosmos SDK will need to be provided. The dependency you will need to use is buf.build/cosmos/cosmos-sdk.

# Generated by buf. DO NOT EDIT.
version: v1
deps:
  - remote: buf.build
    owner: cosmos
    repository: cosmos-proto
    commit: 04467658e59e44bbb22fe568206e1f70
    digest: shake256:73a640bd60e0c523b0f8237ff34eab67c45a38b64bbbde1d80224819d272dbf316ac183526bd245f994af6608b025f5130483d0133c5edd385531326b5990466
  - remote: buf.build
    owner: cosmos
    repository: gogo-proto
    commit: 88ef6483f90f478fb938c37dde52ece3
    digest: shake256:89c45df2aa11e0cff97b0d695436713db3d993d76792e9f8dc1ae90e6ab9a9bec55503d48ceedd6b86069ab07d3041b32001b2bfe0227fa725dd515ff381e5ba
  - remote: buf.build
    owner: googleapis
    repository: googleapis
    commit: 751cbe31638d43a9bfb6162cd2352e67
    digest: shake256:87f55470d9d124e2d1dedfe0231221f4ed7efbc55bc5268917c678e2d9b9c41573a7f9a557f6d8539044524d9fc5ca8fbb7db05eb81379d168285d76b57eb8a4
  - remote: buf.build
    owner: protocolbuffers
    repository: wellknowntypes
    commit: 3ddd61d1f53d485abd3d3a2b47a62b8e
    digest: shake256:9e6799d56700d0470c3723a2fd027e8b4a41a07085a0c90c58e05f6c0038fac9b7a0170acd7692707a849983b1b8189aa33e7b73f91d68157f7136823115546b

buf.yaml

buf.yaml defines the name of your package, which breakage checker to use and how to lint your protobuf files.

# This module represents buf.build/cosmos/cosmos-sdk
version: v1
name: buf.build/cosmos/cosmos-sdk
deps:
  - buf.build/cosmos/cosmos-proto
  - buf.build/cosmos/gogo-proto
  - buf.build/googleapis/googleapis
  - buf.build/protocolbuffers/wellknowntypes
breaking:
  use:
    - FILE
  ignore:
    - testpb
lint:
  use:
    - STANDARD
    - COMMENTS
    - FILE_LOWER_SNAKE_CASE
  except:
    - UNARY_RPC
    - COMMENT_FIELD
    - SERVICE_SUFFIX
    - PACKAGE_VERSION_SUFFIX
    - RPC_REQUEST_STANDARD_NAME
  ignore:
    - tendermint

We use a variety of linters for the Cosmos SDK protobuf files. The repo also checks this in ci.

A reference to the github actions can be found here

name: Protobuf
# Protobuf runs buf (https://buf.build/)

lint and check-breakage
# This workflow is only run when a .proto file has been changed
on:
  pull_request:
    paths:
      - "proto/**"

permissions:
  contents: read

jobs:
  lint:
    runs-on: depot-ubuntu-22.04-4
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-lint-action@v1
        with:
          input: "proto"

  break-check:
    runs-on: depot-ubuntu-22.04-4
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-breaking-action@v1
        with:
          input: "proto"
          against: "https://github.com/${{
    github.repository
}}.git#branch=${{
    github.event.pull_request.base.ref
}},ref=HEAD~1,subdir=proto"

; then buf generate —template buf.gen.gogo.yaml $file fi done done

cd ..

generate codec/testdata proto code

(cd testutil/testdata; buf generate)

generate baseapp test messages

(cd baseapp/testutil; buf generate)

move proto files to the right places

cp -r github.com/cosmos/cosmos-sdk/* ./ rm -rf github.com

go mod tidy

./scripts/protocgen-pulsar.sh

## Buf

[Buf](https://buf.build) is a protobuf tool that abstracts the needs to use the complicated `protoc` toolchain on top of various other things that ensure you are using protobuf in accordance with the majority of the ecosystem. Within the cosmos-sdk repository there are a few files that have a buf prefix. Lets start with the top level and then dive into the various directories.

### Workspace

At the root level directory a workspace is defined using [buf workspaces](https://docs.buf.build/configuration/v1/buf-work-yaml). This helps if there are one or more protobuf containing directories in your project.

Cosmos SDK example:

```go
version: v1
directories:
  - proto

Proto Directory

Next is the proto/ directory where all of our protobuf files live. In here there are many different buf files defined each serving a different purpose.

├── 05-depinject.md
├── buf.gen.gogo.yaml
├── buf.gen.pulsar.yaml
├── buf.gen.swagger.yaml
├── buf.lock
├── buf.md
├── buf.yaml
├── cosmos
└── tendermint

The above diagram all the files and directories within the Cosmos SDK proto/ directory.

buf.gen.gogo.yaml

buf.gen.gogo.yaml defines how the protobuf files should be generated for use with in the module. This file uses gogoproto, a separate generator from the google go-proto generator that makes working with various objects more ergonomic, and it has more performant encode and decode steps

version: v1
plugins:
  - name: gocosmos
    out: ..
    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/gogoproto/types/any
  - name: grpc-gateway
    out: ..
    opt: logtostderr=true,allow_colon_final_segments=true

Example of how to define gen files can be found here

buf.gen.pulsar.yaml

buf.gen.pulsar.yaml defines how protobuf files should be generated using the new golang apiv2 of protobuf. This generator is used instead of the google go-proto generator because it has some extra helpers for Cosmos SDK applications and will have more performant encode and decode than the google go-proto generator. You can follow the development of this generator here.

version: v1
managed:
  enabled: true
  go_package_prefix:
    default: cosmossdk.io/api
    except:
      - buf.build/googleapis/googleapis
      - buf.build/cosmos/gogo-proto
      - buf.build/cosmos/cosmos-proto
    override:
plugins:
  - name: go-pulsar
    out: ../api
    opt: paths=source_relative
  - name: go-grpc
    out: ../api
    opt: paths=source_relative

Example of how to define gen files can be found here

buf.gen.swagger.yaml

buf.gen.swagger.yaml generates the swagger documentation for the query and messages of the chain. This will only define the REST API end points that were defined in the query and msg servers. You can find examples of this here

version: v1
plugins:
  - name: swagger
    out: ../tmp-swagger-gen
    opt: logtostderr=true,fqn_for_swagger_name=true,simple_operation_ids=true

Example of how to define gen files can be found here

buf.lock

This is a autogenerated file based off the dependencies required by the .gen files. There is no need to copy the current one. If you depend on cosmos-sdk proto definitions a new entry for the Cosmos SDK will need to be provided. The dependency you will need to use is buf.build/cosmos/cosmos-sdk.

# Generated by buf. DO NOT EDIT.
version: v1
deps:
  - remote: buf.build
    owner: cosmos
    repository: cosmos-proto
    commit: 04467658e59e44bbb22fe568206e1f70
    digest: shake256:73a640bd60e0c523b0f8237ff34eab67c45a38b64bbbde1d80224819d272dbf316ac183526bd245f994af6608b025f5130483d0133c5edd385531326b5990466
  - remote: buf.build
    owner: cosmos
    repository: gogo-proto
    commit: 88ef6483f90f478fb938c37dde52ece3
    digest: shake256:89c45df2aa11e0cff97b0d695436713db3d993d76792e9f8dc1ae90e6ab9a9bec55503d48ceedd6b86069ab07d3041b32001b2bfe0227fa725dd515ff381e5ba
  - remote: buf.build
    owner: googleapis
    repository: googleapis
    commit: 751cbe31638d43a9bfb6162cd2352e67
    digest: shake256:87f55470d9d124e2d1dedfe0231221f4ed7efbc55bc5268917c678e2d9b9c41573a7f9a557f6d8539044524d9fc5ca8fbb7db05eb81379d168285d76b57eb8a4
  - remote: buf.build
    owner: protocolbuffers
    repository: wellknowntypes
    commit: 3ddd61d1f53d485abd3d3a2b47a62b8e
    digest: shake256:9e6799d56700d0470c3723a2fd027e8b4a41a07085a0c90c58e05f6c0038fac9b7a0170acd7692707a849983b1b8189aa33e7b73f91d68157f7136823115546b

buf.yaml

buf.yaml defines the name of your package, which breakage checker to use and how to lint your protobuf files.

# This module represents buf.build/cosmos/cosmos-sdk
version: v1
name: buf.build/cosmos/cosmos-sdk
deps:
  - buf.build/cosmos/cosmos-proto
  - buf.build/cosmos/gogo-proto
  - buf.build/googleapis/googleapis
  - buf.build/protocolbuffers/wellknowntypes
breaking:
  use:
    - FILE
  ignore:
    - testpb
lint:
  use:
    - STANDARD
    - COMMENTS
    - FILE_LOWER_SNAKE_CASE
  except:
    - UNARY_RPC
    - COMMENT_FIELD
    - SERVICE_SUFFIX
    - PACKAGE_VERSION_SUFFIX
    - RPC_REQUEST_STANDARD_NAME
  ignore:
    - tendermint

We use a variety of linters for the Cosmos SDK protobuf files. The repo also checks this in ci.

A reference to the github actions can be found here

name: Protobuf
# Protobuf runs buf (https://buf.build/)

lint and check-breakage
# This workflow is only run when a .proto file has been changed
on:
  pull_request:
    paths:
      - "proto/**"

permissions:
  contents: read

jobs:
  lint:
    runs-on: depot-ubuntu-22.04-4
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-lint-action@v1
        with:
          input: "proto"

  break-check:
    runs-on: depot-ubuntu-22.04-4
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-breaking-action@v1
        with:
          input: "proto"
          against: "https://github.com/${{
    github.repository
}}.git#branch=${{
    github.event.pull_request.base.ref
}},ref=HEAD~1,subdir=proto"