Protocol Buffers

It is known that Cosmos SDK uses protocol buffers extensively, this document is meant to provide a guide on how it is used in the cosmos-sdk.

It is known that Cosmos SDK uses protocol buffers extensively, this document is meant to provide a guide on how it is used in the cosmos-sdk.

To generate the proto file, the Cosmos SDK uses a docker image, this image is provided to all to use as well. The latest version is ghcr.io/cosmos/proto-builder:0.12.x

Below is the example of the Cosmos SDK’s commands for generating, linting, and formatting protobuf files that can be reused in any applications makefile.

#!/usr/bin/make -f

PACKAGES_NOSIMULATION=$(shell go list ./... | grep -v '/simulation')

PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation')

export VERSION := $(shell echo $(shell git describe --tags --always --match "v*") | sed 's/^v/')

export CMTVERSION := $(shell go list -m github.com/cometbft/cometbft | sed 's:.* ::')

export COMMIT := $(shell git log -1 --format='%H')

LEDGER_ENABLED ?= true
BINDIR ?= $(GOPATH)/bin
BUILDDIR ?= $(CURDIR)/build
SIMAPP = ./simapp
MOCKS_DIR = $(CURDIR)/tests/mocks
HTTPS_GIT := https://github.com/cosmos/cosmos-sdk.git
DOCKER := $(shell which docker)

PROJECT_NAME = $(shell git remote get-url origin | xargs basename -s .git)

# process build tags
build_tags = netgo
    ifeq ($(LEDGER_ENABLED),true)
    ifeq ($(OS),Windows_NT)

GCCEXE = $(shell where gcc.exe 2> NUL)
    ifeq ($(GCCEXE),)
		$(error gcc.exe not installed for ledger support, please install or set LEDGER_ENABLED=false)

else
		build_tags += ledger
	endif
	else
	UNAME_S = $(shell uname -s)
    ifeq ($(UNAME_S),OpenBSD)
		$(warning OpenBSD detected, disabling ledger support (https://github.com/cosmos/cosmos-sdk/issues/1988))

else
		GCC = $(shell command -v gcc 2> /dev/null)
    ifeq ($(GCC),)
			$(error gcc not installed for ledger support, please install or set LEDGER_ENABLED=false)

else
			build_tags += ledger
		endif
	endif
	endif
endif
    ifeq (secp,$(findstring secp,$(COSMOS_BUILD_OPTIONS)))

build_tags += libsecp256k1_sdk
endif
    ifeq (legacy,$(findstring legacy,$(COSMOS_BUILD_OPTIONS)))

build_tags += app_v1
endif
    whitespace :=
whitespace += $(whitespace)
    comma := ,
build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags))

# process linker flags

ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=sim \
		-X github.com/cosmos/cosmos-sdk/version.AppName=simd \
		-X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \
		-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
		-X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)" \
		-X github.com/cometbft/cometbft/version.TMCoreSemVer=$(CMTVERSION)

# DB backend selection
    ifeq (cleveldb,$(findstring cleveldb,$(COSMOS_BUILD_OPTIONS)))

build_tags += gcc
endif
    ifeq (badgerdb,$(findstring badgerdb,$(COSMOS_BUILD_OPTIONS)))

build_tags += badgerdb
endif
# handle rocksdb
    ifeq (rocksdb,$(findstring rocksdb,$(COSMOS_BUILD_OPTIONS)))

CGO_ENABLED=1
  build_tags += rocksdb
endif
# handle boltdb
    ifeq (boltdb,$(findstring boltdb,$(COSMOS_BUILD_OPTIONS)))

build_tags += boltdb
endif
    ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS)))

ldflags += -w -s
endif
ldflags += $(LDFLAGS)
    ldflags := $(strip $(ldflags))

build_tags += $(BUILD_TAGS)

build_tags := $(strip $(build_tags))

BUILD_FLAGS := -tags "$(build_tags)" -ldflags '$(ldflags)'
# check for nostrip option
    ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS)))

BUILD_FLAGS += -trimpath
endif

# Check for debug option
    ifeq (debug,$(findstring debug,$(COSMOS_BUILD_OPTIONS)))

BUILD_FLAGS += -gcflags "all=-N -l"
endif

all: tools build lint test vulncheck

# The below include contains the tools and runsim targets.
include contrib/devtools/Makefile

###############################################################################
###                                  Build                                  ###
###############################################################################

BUILD_TARGETS := build install

build: BUILD_ARGS=-o $(BUILDDIR)/

build-linux-amd64:
	GOOS=linux GOARCH=amd64 LEDGER_ENABLED=false $(MAKE)

build

build-linux-arm64:
	GOOS=linux GOARCH=arm64 LEDGER_ENABLED=false $(MAKE)

build

$(BUILD_TARGETS): go.sum $(BUILDDIR)/
	cd ${
    CURRENT_DIR
}/simapp && go $@ -mod=readonly $(BUILD_FLAGS) $(BUILD_ARGS) ./...

$(BUILDDIR)/:
	mkdir -p $(BUILDDIR)/

cosmovisor:
	$(MAKE) -C tools/cosmovisor cosmovisor

rosetta:
	$(MAKE) -C tools/rosetta rosetta

confix:
	$(MAKE) -C tools/confix confix

hubl:
	$(MAKE) -C tools/hubl hubl

.PHONY: build build-linux-amd64 build-linux-arm64 cosmovisor rosetta confix

mocks: $(MOCKS_DIR)
	@go install github.com/golang/mock/mockgen@v1.6.0
	sh ./scripts/mockgen.sh
.PHONY: mocks

vulncheck: $(BUILDDIR)/
	GOBIN=$(BUILDDIR)

go install golang.org/x/vuln/cmd/govulncheck@latest
	$(BUILDDIR)/govulncheck ./...

$(MOCKS_DIR):
	mkdir -p $(MOCKS_DIR)

distclean: clean tools-clean
clean:
	rm -rf \
	$(BUILDDIR)/ \
	artifacts/ \
	tmp-swagger-gen/ \
	.testnets

.PHONY: distclean clean

###############################################################################
###                          Tools & Dependencies                           ###
###############################################################################

go.sum: go.mod
	echo "Ensure dependencies have not been modified ..." >&2
	go mod verify
	go mod tidy

###############################################################################
###                              Documentation                              ###
###############################################################################

godocs:
	@echo "--> Wait a few seconds and visit http://localhost:6060/pkg/github.com/cosmos/cosmos-sdk/types"
	go install golang.org/x/tools/cmd/godoc@latest
	godoc -http=:6060

build-docs:
	@cd docs && DOCS_DOMAIN=docs.cosmos.network sh ./build-all.sh

.PHONY: build-docs

###############################################################################
###                           Tests & Simulation                            ###
###############################################################################

# make init-simapp initializes a single local node network
# it is useful for testing and development
# Usage: make install && make init-simapp && simd start
# Warning: make init-simapp will remove all data in simapp home directory
init-simapp:
	./scripts/init-simapp.sh

test: test-unit
test-e2e:
	$(MAKE) -C tests test-e2e
test-e2e-cov:
	$(MAKE) -C tests test-e2e-cov
test-integration:
	$(MAKE) -C tests test-integration
test-integration-cov:
	$(MAKE) -C tests test-integration-cov
test-all: test-unit test-e2e test-integration test-ledger-mock test-race

TEST_PACKAGES=./...
TEST_TARGETS := test-unit test-unit-amino test-unit-proto test-ledger-mock test-race test-ledger test-race

# Test runs-specific rules. To add a new test target, just add
# a new rule, customise ARGS or TEST_PACKAGES ad libitum, and
# append the new rule to the TEST_TARGETS list.
test-unit: test_tags += cgo ledger test_ledger_mock norace
test-unit-amino: test_tags += ledger test_ledger_mock test_amino norace
test-ledger: test_tags += cgo ledger norace
test-ledger-mock: test_tags += ledger test_ledger_mock norace
test-race: test_tags += cgo ledger test_ledger_mock
test-race: ARGS=-race
test-race: TEST_PACKAGES=$(PACKAGES_NOSIMULATION)
$(TEST_TARGETS): run-tests

# check-* compiles and collects tests without running them
# note: go test -c doesn't support multiple packages yet (https://github.com/golang/go/issues/15513)

CHECK_TEST_TARGETS := check-test-unit check-test-unit-amino
check-test-unit: test_tags += cgo ledger test_ledger_mock norace
check-test-unit-amino: test_tags += ledger test_ledger_mock test_amino norace
$(CHECK_TEST_TARGETS): EXTRA_ARGS=-run=none
$(CHECK_TEST_TARGETS): run-tests

ARGS += -tags "$(test_tags)"
SUB_MODULES = $(shell find . -type f -name 'go.mod' -print0 | xargs -0 -n1 dirname | sort)

CURRENT_DIR = $(shell pwd)

run-tests:
    ifneq (,$(shell which tparse 2>/dev/null))
	@echo "Starting unit tests"; \
	finalec=0; \
    for module in $(SUB_MODULES); do \
		cd ${
    CURRENT_DIR
}/$module; \
		echo "Running unit tests for $(grep '^module' go.mod)"; \
		go test -mod=readonly -json $(ARGS) $(TEST_PACKAGES) ./... | tparse; \
		ec=$?; \
    if [ "$ec" -ne '0' ]; then finalec=$ec; fi; \
	done; \
	exit $finalec
else
	@echo "Starting unit tests"; \
	finalec=0; \
    for module in $(SUB_MODULES); do \
		cd ${
    CURRENT_DIR
}/$module; \
		echo "Running unit tests for $(grep '^module' go.mod)"; \
		go test -mod=readonly $(ARGS) $(TEST_PACKAGES) ./... ; \
		ec=$?; \
    if [ "$ec" -ne '0' ]; then finalec=$ec; fi; \
	done; \
	exit $finalec
endif

.PHONY: run-tests test test-all $(TEST_TARGETS)

test-sim-nondeterminism:
	@echo "Running non-determinism test..."
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
		-NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h

# Requires an exported plugin. See store/streaming/README.md for documentation.
#
# example:
#   export COSMOS_SDK_ABCI_V1=<path-to-plugin-binary>
#   make test-sim-nondeterminism-streaming
#
# Using the built-in examples:
#   export COSMOS_SDK_ABCI_V1=<path-to-sdk>/store/streaming/abci/examples/file/file
#   make test-sim-nondeterminism-streaming
test-sim-nondeterminism-streaming:
	@echo "Running non-determinism-streaming test..."
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
		-NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h -EnableStreaming=true

test-sim-custom-genesis-fast:
	@echo "Running custom genesis simulation..."
	@echo "By default, ${
    HOME
}/.gaiad/config/genesis.json will be used."
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -run TestFullAppSimulation -Genesis=${
    HOME
}/.gaiad/config/genesis.json \
		-Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -v -timeout 24h

test-sim-import-export: runsim
	@echo "Running application import/export simulation. This may take several minutes..."
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppImportExport

test-sim-after-import: runsim
	@echo "Running application simulation-after-import. This may take several minutes..."
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppSimulationAfterImport

test-sim-custom-genesis-multi-seed: runsim
	@echo "Running multi-seed custom genesis simulation..."
	@echo "By default, ${
    HOME
}/.gaiad/config/genesis.json will be used."
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Genesis=${
    HOME
}/.gaiad/config/genesis.json -SimAppPkg=. -ExitOnFail 400 5 TestFullAppSimulation

test-sim-multi-seed-long: runsim
	@echo "Running long multi-seed application simulation. This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 500 50 TestFullAppSimulation

test-sim-multi-seed-short: runsim
	@echo "Running short multi-seed application simulation. This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 10 TestFullAppSimulation

test-sim-benchmark-invariants:
	@echo "Running simulation invariant benchmarks..."
	cd ${
    CURRENT_DIR
}/simapp && @go test -mod=readonly -benchmem -bench=BenchmarkInvariants -run=^$ \
	-Enabled=true -NumBlocks=1000 -BlockSize=200 \
	-Period=1 -Commit=true -Seed=57 -v -timeout 24h

.PHONY: \
test-sim-nondeterminism \
test-sim-nondeterminism-streaming \
test-sim-custom-genesis-fast \
test-sim-import-export \
test-sim-after-import \
test-sim-custom-genesis-multi-seed \
test-sim-multi-seed-short \
test-sim-multi-seed-long \
test-sim-benchmark-invariants

SIM_NUM_BLOCKS ?= 500
SIM_BLOCK_SIZE ?= 200
SIM_COMMIT ?= true

test-sim-benchmark:
	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -run=^$ $(.) -bench ^BenchmarkFullAppSimulation$  \
		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h

# Requires an exported plugin. See store/streaming/README.md for documentation.
#
# example:
#   export COSMOS_SDK_ABCI_V1=<path-to-plugin-binary>
#   make test-sim-benchmark-streaming
#
# Using the built-in examples:
#   export COSMOS_SDK_ABCI_V1=<path-to-sdk>/store/streaming/abci/examples/file/file
#   make test-sim-benchmark-streaming
test-sim-benchmark-streaming:
	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -run=^$ $(.) -bench ^BenchmarkFullAppSimulation$  \
		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -EnableStreaming=true

test-sim-profile:
	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -benchmem -run=^$ $(.) -bench ^BenchmarkFullAppSimulation$ \
		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out

# Requires an exported plugin. See store/streaming/README.md for documentation.
#
# example:
#   export COSMOS_SDK_ABCI_V1=<path-to-plugin-binary>
#   make test-sim-profile-streaming
#
# Using the built-in examples:
#   export COSMOS_SDK_ABCI_V1=<path-to-sdk>/store/streaming/abci/examples/file/file
#   make test-sim-profile-streaming
test-sim-profile-streaming:
	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
	@cd ${
    CURRENT_DIR
}/simapp && go test -mod=readonly -benchmem -run=^$ $(.) -bench ^BenchmarkFullAppSimulation$ \
		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out -EnableStreaming=true

.PHONY: test-sim-profile test-sim-benchmark

test-rosetta:
	docker build -t rosetta-ci:latest -f contrib/rosetta/rosetta-ci/Dockerfile .
	docker-compose -f contrib/rosetta/docker-compose.yaml up --abort-on-container-exit --exit-code-from test_rosetta --build
.PHONY: test-rosetta

benchmark:
	@go test -mod=readonly -bench=. $(PACKAGES_NOSIMULATION)
.PHONY: benchmark

###############################################################################
###                                Linting                                  ###
###############################################################################

golangci_lint_cmd=golangci-lint
golangci_version=v1.51.2

lint:
	@echo "--> Running linter"
	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(golangci_version)
	@./scripts/go-lint-all.bash --timeout=15m

lint-fix:
	@echo "--> Running linter"
	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(golangci_version)
	@./scripts/go-lint-all.bash --fix

.PHONY: lint lint-fix

###############################################################################
###                                Protobuf                                 ###
###############################################################################

protoVer=0.13.2
protoImageName=ghcr.io/cosmos/proto-builder:$(protoVer)

protoImage=$(DOCKER)

run --rm -v $(CURDIR):/workspace --workdir /workspace $(protoImageName)

proto-all: proto-format proto-lint proto-gen

proto-gen:
	@echo "Generating Protobuf files"
	@$(protoImage)

sh ./scripts/protocgen.sh

proto-swagger-gen:
	@echo "Generating Protobuf Swagger"
	@$(protoImage)

sh ./scripts/protoc-swagger-gen.sh

proto-format:
	@$(protoImage)

find ./ -name "*.proto" -exec clang-format -i {
} \;

proto-lint:
	@$(protoImage)

buf lint --error-format=json

proto-check-breaking:
	@$(protoImage)

buf breaking --against $(HTTPS_GIT)#branch=main

CMT_URL              = https://raw.githubusercontent.com/cometbft/cometbft/v0.38.0-alpha.2/proto/tendermint

CMT_CRYPTO_TYPES     = proto/tendermint/crypto
CMT_ABCI_TYPES       = proto/tendermint/abci
CMT_TYPES            = proto/tendermint/types
CMT_VERSION          = proto/tendermint/version
CMT_LIBS             = proto/tendermint/libs/bits
CMT_P2P              = proto/tendermint/p2p

proto-update-deps:
	@echo "Updating Protobuf dependencies"

	@mkdir -p $(CMT_ABCI_TYPES)
	@curl -sSL $(CMT_URL)/abci/types.proto > $(CMT_ABCI_TYPES)/types.proto

	@mkdir -p $(CMT_VERSION)
	@curl -sSL $(CMT_URL)/version/types.proto > $(CMT_VERSION)/types.proto

	@mkdir -p $(CMT_TYPES)
	@curl -sSL $(CMT_URL)/types/types.proto > $(CMT_TYPES)/types.proto
	@curl -sSL $(CMT_URL)/types/evidence.proto > $(CMT_TYPES)/evidence.proto
	@curl -sSL $(CMT_URL)/types/params.proto > $(CMT_TYPES)/params.proto
	@curl -sSL $(CMT_URL)/types/validator.proto > $(CMT_TYPES)/validator.proto
	@curl -sSL $(CMT_URL)/types/block.proto > $(CMT_TYPES)/block.proto

	@mkdir -p $(CMT_CRYPTO_TYPES)
	@curl -sSL $(CMT_URL)/crypto/proof.proto > $(CMT_CRYPTO_TYPES)/proof.proto
	@curl -sSL $(CMT_URL)/crypto/keys.proto > $(CMT_CRYPTO_TYPES)/keys.proto

	@mkdir -p $(CMT_LIBS)
	@curl -sSL $(CMT_URL)/libs/bits/types.proto > $(CMT_LIBS)/types.proto

	@mkdir -p $(CMT_P2P)
	@curl -sSL $(CMT_URL)/p2p/types.proto > $(CMT_P2P)/types.proto

	$(DOCKER)

run --rm -v $(CURDIR)/proto:/workspace --workdir /workspace $(protoImageName)

buf mod update

.PHONY: proto-all proto-gen proto-swagger-gen proto-format proto-lint proto-check-breaking proto-update-deps

###############################################################################
###                                Localnet                                 ###
###############################################################################

localnet-build-env:
	$(MAKE) -C contrib/images simd-env
localnet-build-dlv:
	$(MAKE) -C contrib/images simd-dlv

localnet-build-nodes:
	$(DOCKER)

run --rm -v $(CURDIR)/.testnets:/data cosmossdk/simd \
			  testnet init-files --v 4 -o /data --starting-ip-address 192.168.10.2 --keyring-backend=test
	docker-compose up -d

localnet-stop:
	docker-compose down

# localnet-start will run a 4-node testnet locally. The nodes are
# based off the docker images in: ./contrib/images/simd-env
localnet-start: localnet-stop localnet-build-env localnet-build-nodes

# localnet-debug will run a 4-node testnet locally in debug mode
# you can read more about the debug mode here: ./contrib/images/simd-dlv/README.md
localnet-debug: localnet-stop localnet-build-dlv localnet-build-nodes

.PHONY: localnet-start localnet-stop localnet-debug localnet-build-env localnet-build-dlv localnet-build-nodes

###############################################################################
###                                rosetta                                  ###
###############################################################################
# builds rosetta test data dir
rosetta-data:
	-docker container rm data_dir_build
	docker build -t rosetta-ci:latest -f contrib/rosetta/rosetta-ci/Dockerfile .
	docker run --name data_dir_build -t rosetta-ci:latest sh /rosetta/data.sh
	docker cp data_dir_build:/tmp/data.tar.gz "$(CURDIR)/contrib/rosetta/rosetta-ci/data.tar.gz"
	docker container rm data_dir_build
.PHONY: rosetta-data

The script used to generate the protobuf files can be found in the scripts/ directory.

#!/usr/bin/env bash

# How to run manually:
# docker build --pull --rm -f "contrib/devtools/Dockerfile" -t cosmossdk-proto:latest "contrib/devtools"
# docker run --rm -v $(pwd):/workspace --workdir /workspace cosmossdk-proto sh ./scripts/protocgen.sh

set -e

echo "Generating gogo proto code"
cd proto
proto_dirs=$(find ./cosmos ./amino -path -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq)
for dir in $proto_dirs; do
  for file in $(find "${dir}" -maxdepth 1 -name '*.proto'); do
    # this regex checks if a proto file has its go_package set to cosmossdk.io/api/...
    # gogo proto files SHOULD ONLY be generated if this is false
    # we don't want gogo proto to run for proto files which are natively built for google.golang.org/protobuf
    if grep -q "option go_package" "$file" && grep -H -o -c 'option go_package.*cosmossdk.io/api' "$file" | grep -q ':0

Buf

Buf is a protobuf tool that abstracts the needs to use the complicated protoc toolchain on top of various other things that ensure you are using protobuf in accordance with the majority of the ecosystem. Within the cosmos-sdk repository there are a few files that have a buf prefix. Lets start with the top level and then dive into the various directories.

Workspace

At the root level directory a workspace is defined using buf workspaces. This helps if there are one or more protobuf containing directories in your project.

Cosmos SDK example:

version: v1
directories:
  - proto

Proto Directory

Next is the proto/ directory where all of our protobuf files live. In here there are many different buf files defined each serving a different purpose.

├── README.md
├── buf.gen.gogo.yaml
├── buf.gen.pulsar.yaml
├── buf.gen.swagger.yaml
├── buf.lock
├── buf.md
├── buf.yaml
├── cosmos
└── tendermint

The above diagram all the files and directories within the Cosmos SDK proto/ directory.

buf.gen.gogo.yaml

buf.gen.gogo.yaml defines how the protobuf files should be generated for use with in the module. This file uses gogoproto, a separate generator from the google go-proto generator that makes working with various objects more ergonomic, and it has more performant encode and decode steps

version: v1
plugins:
  - name: gocosmos
    out: ..
    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/gogoproto/types/any
  - name: grpc-gateway
    out: ..
    opt: logtostderr=true,allow_colon_final_segments=true

Example of how to define gen files can be found here

buf.gen.pulsar.yaml

buf.gen.pulsar.yaml defines how protobuf files should be generated using the new golang apiv2 of protobuf. This generator is used instead of the google go-proto generator because it has some extra helpers for Cosmos SDK applications and will have more performant encode and decode than the google go-proto generator. You can follow the development of this generator here.

version: v1
managed:
  enabled: true
  go_package_prefix:
    default: cosmossdk.io/api
    except:
      - buf.build/googleapis/googleapis
      - buf.build/cosmos/gogo-proto
      - buf.build/cosmos/cosmos-proto
    override:
plugins:
  - name: go-pulsar
    out: ../api
    opt: paths=source_relative
  - name: go-grpc
    out: ../api
    opt: paths=source_relative

Example of how to define gen files can be found here

buf.gen.swagger.yaml

buf.gen.swagger.yaml generates the swagger documentation for the query and messages of the chain. This will only define the REST API end points that were defined in the query and msg servers. You can find examples of this here

version: v1
plugins:
  - name: swagger
    out: ../tmp-swagger-gen
    opt: logtostderr=true,fqn_for_swagger_name=true,simple_operation_ids=true

Example of how to define gen files can be found here

buf.lock

This is an autogenerated file based off the dependencies required by the .gen files. There is no need to copy the current one. If you depend on cosmos-sdk proto definitions a new entry for the Cosmos SDK will need to be provided. The dependency you will need to use is buf.build/cosmos/cosmos-sdk.

# Generated by buf. DO NOT EDIT.
version: v1
deps:
  - remote: buf.build
    owner: cosmos
    repository: cosmos-proto
    commit: 04467658e59e44bbb22fe568206e1f70
    digest: shake256:73a640bd60e0c523b0f8237ff34eab67c45a38b64bbbde1d80224819d272dbf316ac183526bd245f994af6608b025f5130483d0133c5edd385531326b5990466
  - remote: buf.build
    owner: cosmos
    repository: gogo-proto
    commit: 88ef6483f90f478fb938c37dde52ece3
    digest: shake256:89c45df2aa11e0cff97b0d695436713db3d993d76792e9f8dc1ae90e6ab9a9bec55503d48ceedd6b86069ab07d3041b32001b2bfe0227fa725dd515ff381e5ba
  - remote: buf.build
    owner: googleapis
    repository: googleapis
    commit: 751cbe31638d43a9bfb6162cd2352e67
    digest: shake256:87f55470d9d124e2d1dedfe0231221f4ed7efbc55bc5268917c678e2d9b9c41573a7f9a557f6d8539044524d9fc5ca8fbb7db05eb81379d168285d76b57eb8a4
  - remote: buf.build
    owner: protocolbuffers
    repository: wellknowntypes
    commit: 3ddd61d1f53d485abd3d3a2b47a62b8e
    digest: shake256:9e6799d56700d0470c3723a2fd027e8b4a41a07085a0c90c58e05f6c0038fac9b7a0170acd7692707a849983b1b8189aa33e7b73f91d68157f7136823115546b

buf.yaml

buf.yaml defines the name of your package, which breakage checker to use and how to lint your protobuf files.

# This module represents buf.build/cosmos/cosmos-sdk
version: v1
name: buf.build/cosmos/cosmos-sdk
deps:
  - buf.build/cosmos/cosmos-proto
  - buf.build/cosmos/gogo-proto
  - buf.build/googleapis/googleapis
  - buf.build/protocolbuffers/wellknowntypes
breaking:
  use:
    - FILE
  ignore:
    - testpb
lint:
  use:
    - STANDARD
    - COMMENTS
    - FILE_LOWER_SNAKE_CASE
  except:
    - UNARY_RPC
    - COMMENT_FIELD
    - SERVICE_SUFFIX
    - PACKAGE_VERSION_SUFFIX
    - RPC_REQUEST_STANDARD_NAME
  ignore:
    - tendermint

We use a variety of linters for the Cosmos SDK protobuf files. The repo also checks this in ci.

A reference to the github actions can be found here

name: Protobuf
# Protobuf runs buf (https://buf.build/)

lint and check-breakage
# This workflow is only run when a .proto file has been changed
on:
  pull_request:
    paths:
      - "proto/**"

permissions:
  contents: read

jobs:
  lint:
    runs-on: depot-ubuntu-22.04-4
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-lint-action@v1
        with:
          input: "proto"

  break-check:
    runs-on: depot-ubuntu-22.04-4
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-breaking-action@v1
        with:
          input: "proto"
          against: "https://github.com/${{
    github.repository
}}.git#branch=${{
    github.event.pull_request.base.ref
}},ref=HEAD~1,subdir=proto"

; then buf generate —template buf.gen.gogo.yaml $file fi done done

cd ..

generate codec/testdata proto code

(cd testutil/testdata; buf generate)

generate baseapp test messages

(cd baseapp/testutil; buf generate)

move proto files to the right places

cp -r github.com/cosmos/cosmos-sdk/* ./ cp -r cosmossdk.io/** ./ rm -rf github.com cosmossdk.io

go mod tidy

./scripts/protocgen-pulsar.sh

## Buf

[Buf](https://buf.build) is a protobuf tool that abstracts the needs to use the complicated `protoc` toolchain on top of various other things that ensure you are using protobuf in accordance with the majority of the ecosystem. Within the cosmos-sdk repository there are a few files that have a buf prefix. Lets start with the top level and then dive into the various directories.

### Workspace

At the root level directory a workspace is defined using [buf workspaces](https://docs.buf.build/configuration/v1/buf-work-yaml). This helps if there are one or more protobuf containing directories in your project.

Cosmos SDK example:

```go
version: v1
directories:
  - proto

Proto Directory

Next is the proto/ directory where all of our protobuf files live. In here there are many different buf files defined each serving a different purpose.

├── README.md
├── buf.gen.gogo.yaml
├── buf.gen.pulsar.yaml
├── buf.gen.swagger.yaml
├── buf.lock
├── buf.md
├── buf.yaml
├── cosmos
└── tendermint

The above diagram all the files and directories within the Cosmos SDK proto/ directory.

buf.gen.gogo.yaml

buf.gen.gogo.yaml defines how the protobuf files should be generated for use with in the module. This file uses gogoproto, a separate generator from the google go-proto generator that makes working with various objects more ergonomic, and it has more performant encode and decode steps

version: v1
plugins:
  - name: gocosmos
    out: ..
    opt: plugins=grpc,Mgoogle/protobuf/any.proto=github.com/cosmos/gogoproto/types/any
  - name: grpc-gateway
    out: ..
    opt: logtostderr=true,allow_colon_final_segments=true

Example of how to define gen files can be found here

buf.gen.pulsar.yaml

buf.gen.pulsar.yaml defines how protobuf files should be generated using the new golang apiv2 of protobuf. This generator is used instead of the google go-proto generator because it has some extra helpers for Cosmos SDK applications and will have more performant encode and decode than the google go-proto generator. You can follow the development of this generator here.

version: v1
managed:
  enabled: true
  go_package_prefix:
    default: cosmossdk.io/api
    except:
      - buf.build/googleapis/googleapis
      - buf.build/cosmos/gogo-proto
      - buf.build/cosmos/cosmos-proto
    override:
plugins:
  - name: go-pulsar
    out: ../api
    opt: paths=source_relative
  - name: go-grpc
    out: ../api
    opt: paths=source_relative

Example of how to define gen files can be found here

buf.gen.swagger.yaml

buf.gen.swagger.yaml generates the swagger documentation for the query and messages of the chain. This will only define the REST API end points that were defined in the query and msg servers. You can find examples of this here

version: v1
plugins:
  - name: swagger
    out: ../tmp-swagger-gen
    opt: logtostderr=true,fqn_for_swagger_name=true,simple_operation_ids=true

Example of how to define gen files can be found here

buf.lock

This is an autogenerated file based off the dependencies required by the .gen files. There is no need to copy the current one. If you depend on cosmos-sdk proto definitions a new entry for the Cosmos SDK will need to be provided. The dependency you will need to use is buf.build/cosmos/cosmos-sdk.

# Generated by buf. DO NOT EDIT.
version: v1
deps:
  - remote: buf.build
    owner: cosmos
    repository: cosmos-proto
    commit: 04467658e59e44bbb22fe568206e1f70
    digest: shake256:73a640bd60e0c523b0f8237ff34eab67c45a38b64bbbde1d80224819d272dbf316ac183526bd245f994af6608b025f5130483d0133c5edd385531326b5990466
  - remote: buf.build
    owner: cosmos
    repository: gogo-proto
    commit: 88ef6483f90f478fb938c37dde52ece3
    digest: shake256:89c45df2aa11e0cff97b0d695436713db3d993d76792e9f8dc1ae90e6ab9a9bec55503d48ceedd6b86069ab07d3041b32001b2bfe0227fa725dd515ff381e5ba
  - remote: buf.build
    owner: googleapis
    repository: googleapis
    commit: 751cbe31638d43a9bfb6162cd2352e67
    digest: shake256:87f55470d9d124e2d1dedfe0231221f4ed7efbc55bc5268917c678e2d9b9c41573a7f9a557f6d8539044524d9fc5ca8fbb7db05eb81379d168285d76b57eb8a4
  - remote: buf.build
    owner: protocolbuffers
    repository: wellknowntypes
    commit: 3ddd61d1f53d485abd3d3a2b47a62b8e
    digest: shake256:9e6799d56700d0470c3723a2fd027e8b4a41a07085a0c90c58e05f6c0038fac9b7a0170acd7692707a849983b1b8189aa33e7b73f91d68157f7136823115546b

buf.yaml

buf.yaml defines the name of your package, which breakage checker to use and how to lint your protobuf files.

# This module represents buf.build/cosmos/cosmos-sdk
version: v1
name: buf.build/cosmos/cosmos-sdk
deps:
  - buf.build/cosmos/cosmos-proto
  - buf.build/cosmos/gogo-proto
  - buf.build/googleapis/googleapis
  - buf.build/protocolbuffers/wellknowntypes
breaking:
  use:
    - FILE
  ignore:
    - testpb
lint:
  use:
    - STANDARD
    - COMMENTS
    - FILE_LOWER_SNAKE_CASE
  except:
    - UNARY_RPC
    - COMMENT_FIELD
    - SERVICE_SUFFIX
    - PACKAGE_VERSION_SUFFIX
    - RPC_REQUEST_STANDARD_NAME
  ignore:
    - tendermint

We use a variety of linters for the Cosmos SDK protobuf files. The repo also checks this in ci.

A reference to the github actions can be found here

name: Protobuf
# Protobuf runs buf (https://buf.build/)

lint and check-breakage
# This workflow is only run when a .proto file has been changed
on:
  pull_request:
    paths:
      - "proto/**"

permissions:
  contents: read

jobs:
  lint:
    runs-on: depot-ubuntu-22.04-4
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-lint-action@v1
        with:
          input: "proto"

  break-check:
    runs-on: depot-ubuntu-22.04-4
    steps:
      - uses: actions/checkout@v5
      - uses: bufbuild/buf-setup-action@v1.50.0
      - uses: bufbuild/buf-breaking-action@v1
        with:
          input: "proto"
          against: "https://github.com/${{
    github.repository
}}.git#branch=${{
    github.event.pull_request.base.ref
}},ref=HEAD~1,subdir=proto"