From 2dfca373f3dfc07c1bd91cbc0272f87f6619ee58 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Mon, 19 Sep 2022 11:34:47 +0200
Subject: [PATCH] append nodes to builder support
Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
---
.github/workflows/ci.yml | 33 ++++++++++++++++++++
README.md | 3 ++
__tests__/context.test.ts | 52 ++++++++++++++++++++++++++++++++
action.yml | 3 ++
docs/advanced/append-nodes.md | 56 ++++++++++++++++++++++++++++++++++
docs/advanced/auth.md | 5 ---
jest.config.ts | 7 +++--
package.json | 2 ++
src/context.ts | 57 +++++++++++++++++++++++++++++++----
src/main.ts | 27 ++++++++++++++++-
src/nodes.ts | 13 ++++++++
yarn.lock | 5 +++
12 files changed, 249 insertions(+), 14 deletions(-)
create mode 100644 docs/advanced/append-nodes.md
create mode 100644 src/nodes.ts
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 505418a..04647f3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -305,9 +305,13 @@ jobs:
platforms: ${{ matrix.qemu-platforms }}
-
name: Set up Docker Buildx
+ id: buildx
uses: ./
with:
version: ${{ matrix.buildx-version }}
+ -
+ name: List builder platforms
+ run: echo ${{ steps.buildx.outputs.platforms }}
build-ref:
runs-on: ubuntu-latest
@@ -416,3 +420,32 @@ jobs:
echo "::error::Should have failed"
exit 1
fi
+
+ append:
+ runs-on: ubuntu-latest
+ steps:
+ -
+ name: Checkout
+ uses: actions/checkout@v3
+ -
+ name: Create dummy contexts
+ run: |
+ docker context create ctxbuilder2
+ docker context create ctxbuilder3
+ -
+ name: Set up Docker Buildx
+ id: buildx
+ uses: ./
+ with:
+ append: |
+ - name: builder2
+ endpoint: ctxbuilder2
+ platforms: linux/amd64
+ driver-opts:
+ - image=moby/buildkit:master
+ - network=host
+ - endpoint: ctxbuilder3
+ platforms: linux/arm64
+ -
+ name: List builder platforms
+ run: echo ${{ steps.buildx.outputs.platforms }}
diff --git a/README.md b/README.md
index 272e28f..3b10951 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,7 @@ ___
* [Usage](#usage)
* [Advanced usage](#advanced-usage)
* [Authentication support](docs/advanced/auth.md)
+ * [Append additional nodes to the builder](docs/advanced/append-nodes.md)
* [Install by default](docs/advanced/install-default.md)
* [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
* [Standalone mode](docs/advanced/standalone.md)
@@ -61,6 +62,7 @@ jobs:
## Advanced usage
* [Authentication support](docs/advanced/auth.md)
+* [Append additional nodes to the builder](docs/advanced/append-nodes.md)
* [Install by default](docs/advanced/install-default.md)
* [BuildKit daemon configuration](docs/advanced/buildkit-config.md)
* [Standalone mode](docs/advanced/standalone.md)
@@ -82,6 +84,7 @@ Following inputs can be used as `step.with` keys
| `endpoint` | String | [Optional address for docker socket](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) or context from `docker context ls` |
| `config`¹ | String | [BuildKit config file](https://docs.docker.com/engine/reference/commandline/buildx_create/#config) |
| `config-inline`¹ | String | Same as `config` but inline |
+| `append` | YAML | [Append additional nodes](docs/advanced/append-nodes.md) to the builder |
> * ¹ `config` and `config-inline` are mutually exclusive
diff --git a/__tests__/context.test.ts b/__tests__/context.test.ts
index cf1babf..4c4605e 100644
--- a/__tests__/context.test.ts
+++ b/__tests__/context.test.ts
@@ -4,6 +4,7 @@ import * as os from 'os';
import * as path from 'path';
import * as uuid from 'uuid';
import * as context from '../src/context';
+import * as nodes from '../src/nodes';
const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'docker-setup-buildx-')).split(path.sep).join(path.posix.sep);
jest.spyOn(context, 'tmpDir').mockImplementation((): string => {
@@ -103,6 +104,57 @@ describe('getCreateArgs', () => {
);
});
+describe('getAppendArgs', () => {
+ beforeEach(() => {
+ process.env = Object.keys(process.env).reduce((object, key) => {
+ if (!key.startsWith('INPUT_')) {
+ object[key] = process.env[key];
+ }
+ return object;
+ }, {});
+ });
+
+ // prettier-ignore
+ test.each([
+ [
+ 0,
+ new Map<string, string>([
+ ['install', 'false'],
+ ['use', 'true'],
+ ]),
+ {
+ "name": "aws_graviton2",
+ "endpoint": "ssh://me@graviton2",
+ "driver-opts": [
+ "image=moby/buildkit:latest"
+ ],
+ "buildkitd-flags": "--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host",
+ "platforms": "linux/arm64"
+ },
+ [
+ 'create',
+ '--name', 'builder-9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d',
+ '--append',
+ '--node', 'aws_graviton2',
+ '--driver-opt', 'image=moby/buildkit:latest',
+ '--buildkitd-flags', '--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host',
+ '--platform', 'linux/arm64',
+ 'ssh://me@graviton2'
+ ]
+ ]
+ ])(
+ '[%d] given %p as inputs, returns %p',
+ async (num: number, inputs: Map<string, string>, node: nodes.Node, expected: Array<string>) => {
+ inputs.forEach((value: string, name: string) => {
+ setInput(name, value);
+ });
+ const inp = await context.getInputs();
+ const res = await context.getAppendArgs(inp, node, '0.9.0');
+ expect(res).toEqual(expected);
+ }
+ );
+});
+
describe('getInputList', () => {
it('handles single line correctly', async () => {
await setInput('foo', 'bar');
diff --git a/action.yml b/action.yml
index a36fa4f..4a5a739 100644
--- a/action.yml
+++ b/action.yml
@@ -38,6 +38,9 @@ inputs:
config-inline:
description: 'Inline BuildKit config'
required: false
+ append:
+ description: 'Append additional nodes to the builder'
+ required: false
outputs:
name:
diff --git a/docs/advanced/append-nodes.md b/docs/advanced/append-nodes.md
new file mode 100644
index 0000000..2753519
--- /dev/null
+++ b/docs/advanced/append-nodes.md
@@ -0,0 +1,56 @@
+# Append additional nodes to the builder
+
+Buildx also supports running builds on multiple machines. This is useful for
+building [multi-platform images](https://docs.docker.com/build/building/multi-platform/)
+on native nodes for more complicated cases that are not handled by QEMU and
+generally have better performance or for distributing the build across multiple
+machines.
+
+You can append nodes to the builder that is going to be created with the
+`append` input in the form of a YAML string document to remove limitations
+intrinsically linked to GitHub Actions (only string format is handled in the
+input fields):
+
+| Name | Type | Description |
+|-------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `name` | String | [Name of the node](https://docs.docker.com/engine/reference/commandline/buildx_create/#node). If empty, it is the name of the builder it belongs to, with an index number suffix. This is useful to set it if you want to modify/remove a node in an underlying step of you workflow. |
+| `endpoint` | String | [Docker context or endpoint](https://docs.docker.com/engine/reference/commandline/buildx_create/#description) of the node to add to the builder |
+| `driver-opts` | List | List of additional [driver-specific options](https://docs.docker.com/engine/reference/commandline/buildx_create/#driver-opt) |
+| `buildkitd-flags` | String | [Flags for buildkitd](https://docs.docker.com/engine/reference/commandline/buildx_create/#buildkitd-flags) daemon |
+| `platforms` | String | Fixed [platforms](https://docs.docker.com/engine/reference/commandline/buildx_create/#platform) for the node. If not empty, values take priority over the detected ones. |
+
+Here is an example using remote nodes with the [`remote` driver](https://docs.docker.com/build/building/drivers/remote/)
+and [TLS authentication](auth.md#tls-authentication):
+
+```yaml
+name: ci
+
+on:
+ push:
+
+jobs:
+ buildx:
+ runs-on: ubuntu-latest
+ steps:
+ -
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+ with:
+ driver: remote
+ endpoint: tcp://oneprovider:1234
+ append: |
+ - endpoint: tcp://graviton2:1234
+ platforms: linux/arm64
+ - endpoint: tcp://linuxone:1234
+ platforms: linux/s390x
+ env:
+ BUILDER_NODE_0_AUTH_TLS_CACERT: ${{ secrets.ONEPROVIDER_CA }}
+ BUILDER_NODE_0_AUTH_TLS_CERT: ${{ secrets.ONEPROVIDER_CERT }}
+ BUILDER_NODE_0_AUTH_TLS_KEY: ${{ secrets.ONEPROVIDER_KEY }}
+ BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.GRAVITON2_CA }}
+ BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.GRAVITON2_CERT }}
+ BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.GRAVITON2_KEY }}
+ BUILDER_NODE_2_AUTH_TLS_CACERT: ${{ secrets.LINUXONE_CA }}
+ BUILDER_NODE_2_AUTH_TLS_CERT: ${{ secrets.LINUXONE_CERT }}
+ BUILDER_NODE_2_AUTH_TLS_KEY: ${{ secrets.LINUXONE_KEY }}
+```
diff --git a/docs/advanced/auth.md b/docs/advanced/auth.md
index 871fb47..7a1892e 100644
--- a/docs/advanced/auth.md
+++ b/docs/advanced/auth.md
@@ -41,11 +41,6 @@ the node in the list of nodes:
* `BUILDER_NODE_<idx>_AUTH_TLS_CERT`
* `BUILDER_NODE_<idx>_AUTH_TLS_KEY`
-> **Note**
->
-> The index is always `0` at the moment as we don't support (yet) appending new
-> nodes with this action.
-
```yaml
name: ci
diff --git a/jest.config.ts b/jest.config.ts
index ebf22a5..7ff4e46 100644
--- a/jest.config.ts
+++ b/jest.config.ts
@@ -1,10 +1,13 @@
module.exports = {
clearMocks: true,
moduleFileExtensions: ['js', 'ts'],
- setupFiles: ["dotenv/config"],
+ setupFiles: ['dotenv/config'],
testMatch: ['**/*.test.ts'],
transform: {
'^.+\\.ts$': 'ts-jest'
},
+ moduleNameMapper: {
+ '^csv-parse/sync': '<rootDir>/node_modules/csv-parse/dist/cjs/sync.cjs'
+ },
verbose: true
-}
+};
diff --git a/package.json b/package.json
index 4d05c72..a8188a6 100644
--- a/package.json
+++ b/package.json
@@ -31,6 +31,8 @@
"@actions/exec": "^1.1.1",
"@actions/http-client": "^2.0.1",
"@actions/tool-cache": "^2.0.1",
+ "csv-parse": "^5.1.0",
+ "js-yaml": "^4.1.0",
"semver": "^7.3.7",
"tmp": "^0.2.1",
"uuid": "^9.0.0"
diff --git a/src/context.ts b/src/context.ts
index 6dcb860..e8861d3 100644
--- a/src/context.ts
+++ b/src/context.ts
@@ -3,7 +3,9 @@ import * as os from 'os';
import path from 'path';
import * as tmp from 'tmp';
import * as uuid from 'uuid';
+import {parse} from 'csv-parse/sync';
import * as buildx from './buildx';
+import * as nodes from './nodes';
import * as core from '@actions/core';
let _tmpDir: string;
@@ -32,6 +34,7 @@ export interface Inputs {
endpoint: string;
config: string;
configInline: string;
+ append: string;
}
export async function getInputs(): Promise<Inputs> {
@@ -45,7 +48,8 @@ export async function getInputs(): Promise<Inputs> {
use: core.getBooleanInput('use'),
endpoint: core.getInput('endpoint'),
config: core.getInput('config'),
- configInline: core.getInput('config-inline')
+ configInline: core.getInput('config-inline'),
+ append: core.getInput('append')
};
}
@@ -79,6 +83,28 @@ export async function getCreateArgs(inputs: Inputs, buildxVersion: string): Prom
return args;
}
+export async function getAppendArgs(inputs: Inputs, node: nodes.Node, buildxVersion: string): Promise<Array<string>> {
+ const args: Array<string> = ['create', '--name', inputs.name, '--append'];
+ if (node.name) {
+ args.push('--node', node.name);
+ }
+ if (node['driver-opts'] && buildx.satisfies(buildxVersion, '>=0.3.0')) {
+ await asyncForEach(node['driver-opts'], async driverOpt => {
+ args.push('--driver-opt', driverOpt);
+ });
+ if (inputs.driver != 'remote' && node['buildkitd-flags']) {
+ args.push('--buildkitd-flags', node['buildkitd-flags']);
+ }
+ }
+ if (node.platforms) {
+ args.push('--platform', node.platforms);
+ }
+ if (node.endpoint) {
+ args.push(node.endpoint);
+ }
+ return args;
+}
+
export async function getInspectArgs(inputs: Inputs, buildxVersion: string): Promise<Array<string>> {
const args: Array<string> = ['inspect', '--bootstrap'];
if (buildx.satisfies(buildxVersion, '>=0.4.0')) {
@@ -88,14 +114,33 @@ export async function getInspectArgs(inputs: Inputs, buildxVersion: string): Pro
}
export async function getInputList(name: string, ignoreComma?: boolean): Promise<string[]> {
+ const res: Array<string> = [];
+
const items = core.getInput(name);
if (items == '') {
- return [];
+ return res;
}
- return items
- .split(/\r?\n/)
- .filter(x => x)
- .reduce<string[]>((acc, line) => acc.concat(!ignoreComma ? line.split(',').filter(x => x) : line).map(pat => pat.trim()), []);
+
+ const records = parse(items, {
+ columns: false,
+ relaxQuotes: true,
+ comment: '#',
+ relaxColumnCount: true,
+ skipEmptyLines: true
+ });
+
+ for (const record of records as Array<string[]>) {
+ if (record.length == 1) {
+ res.push(record[0]);
+ continue;
+ } else if (!ignoreComma) {
+ res.push(...record);
+ continue;
+ }
+ res.push(record.join(','));
+ }
+
+ return res.filter(item => item).map(pat => pat.trim());
}
export const asyncForEach = async (array, callback) => {
diff --git a/src/main.ts b/src/main.ts
index 003e7cf..46f54d1 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -5,6 +5,7 @@ import * as auth from './auth';
import * as buildx from './buildx';
import * as context from './context';
import * as docker from './docker';
+import * as nodes from './nodes';
import * as stateHelper from './state-helper';
import * as util from './util';
import * as core from '@actions/core';
@@ -71,6 +72,21 @@ async function run(): Promise<void> {
core.endGroup();
}
+ if (inputs.append) {
+ core.startGroup(`Appending node(s) to builder`);
+ let nodeIndex = 1;
+ for (const node of nodes.Parse(inputs.append)) {
+ const authOpts = auth.setCredentials(credsdir, nodeIndex, inputs.driver, node.endpoint || '');
+ if (authOpts.length > 0) {
+ node['driver-opts'] = [...(node['driver-opts'] || []), ...authOpts];
+ }
+ const appendCmd = buildx.getCommand(await context.getAppendArgs(inputs, node, buildxVersion), standalone);
+ await exec.exec(appendCmd.commandLine, appendCmd.args);
+ nodeIndex++;
+ }
+ core.endGroup();
+ }
+
core.startGroup(`Booting builder`);
const inspectCmd = buildx.getCommand(await context.getInspectArgs(inputs, buildxVersion), standalone);
await exec.exec(inspectCmd.commandLine, inspectCmd.args);
@@ -88,9 +104,18 @@ async function run(): Promise<void> {
core.startGroup(`Inspect builder`);
const builder = await buildx.inspect(inputs.name, standalone);
const firstNode = builder.nodes[0];
+ const reducedPlatforms: Array<string> = [];
+ for (const node of builder.nodes) {
+ for (const platform of node.platforms?.split(',') || []) {
+ if (reducedPlatforms.indexOf(platform) > -1) {
+ continue;
+ }
+ reducedPlatforms.push(platform);
+ }
+ }
core.info(JSON.stringify(builder, undefined, 2));
core.setOutput('driver', builder.driver);
- core.setOutput('platforms', firstNode.platforms);
+ core.setOutput('platforms', reducedPlatforms.join(','));
core.setOutput('nodes', JSON.stringify(builder.nodes, undefined, 2));
core.setOutput('endpoint', firstNode.endpoint); // TODO: deprecated, to be removed in a later version
core.setOutput('status', firstNode.status); // TODO: deprecated, to be removed in a later version
diff --git a/src/nodes.ts b/src/nodes.ts
new file mode 100644
index 0000000..60443e8
--- /dev/null
+++ b/src/nodes.ts
@@ -0,0 +1,13 @@
+import * as yaml from 'js-yaml';
+
+export type Node = {
+ name?: string;
+ endpoint?: string;
+ 'driver-opts'?: Array<string>;
+ 'buildkitd-flags'?: string;
+ platforms?: string;
+};
+
+export function Parse(data: string): Node[] {
+ return yaml.load(data) as Node[];
+}
diff --git a/yarn.lock b/yarn.lock
index 3bdb949..9ee75b2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1450,6 +1450,11 @@ cssstyle@^2.3.0:
dependencies:
cssom "~0.3.6"
+csv-parse@^5.1.0:
+ version "5.3.0"
+ resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.3.0.tgz#85cc02fc9d1c89bd1b02e69069c960f8b8064322"
+ integrity sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==
+
data-urls@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b"