feat: add support for gpt-4o tokenizer encoding (#3) #12

	name: Go

	on:
	push:
	branches: [ main ]
	pull_request:
	branches: [ main ]

	jobs:

	build:
	name: Build
	runs-on: ubuntu-latest
	steps:

	- name: Set up Go 1.x
	uses: actions/setup-go@v4
	with:
	go-version: ^1.19

	- name: Check out code into the Go module directory
	uses: actions/checkout@v3

	- name: use cache
	uses: actions/cache@v3
	with:
	path: ~/go/pkg/mod
	key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-

	- name: Download Modules
	if: steps.cache.outputs.cache-hit != 'true'
	run: go mod download

	- name: Build
	run: go build -o ./bin/ctoc cmd/ctoc/main.go

	- name: Check imports
	shell: bash
	run: \|
	export PATH=$(go env GOPATH)/bin:$PATH
	go get golang.org/x/tools/cmd/goimports
	diff -u <(echo -n) <(goimports -d .)

	- name: Test
	run: go test -v

Provide feedback