diff --git a/.phpstorm.meta.php/options.meta.php b/.phpstorm.meta.php/options.meta.php index 845f5d60..9430456b 100644 --- a/.phpstorm.meta.php/options.meta.php +++ b/.phpstorm.meta.php/options.meta.php @@ -11,7 +11,8 @@ registerArgumentsSet( 'option_names', - 'docker.client.path', 'docker.socket.path', 'script.exit-early', 'step.clone-path' + 'docker.client.path', 'docker.socket.path', 'script.bash-runner', 'script.exit-early', 'script.runner', + 'step.clone-path' ); expectedArguments( diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e1e025..935faa27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,20 @@ The format is based on [Keep a Changelog] and Pipelines adheres to [Semantic Versioning]: https://semver.org/spec/v2.0.0.html ## [unreleased] +### Add +- Run step scripts with `/bin/bash` if it is available; disable with + `script.bash-runner=false` for previous behaviour (#17) (thanks + [Tim Clephas]) +- `script.runner` configuration parameter to change `/bin/sh`, the + default script runner +### Change +- `-c =true` and `false` are now supported for boolean configuration + parameter `` ### Fix - Fix cache and data directory creation mode, limit to user access only +[Tim Clephas]: https://github.com/Timple + ## [0.0.65] - 2022-04-24 ### Add - `--show` and `--show-pipelines`: Annotate steps' conditions with `*C` diff --git a/README.md b/README.md index 0731c5e2..8a98822f 100644 --- a/README.md +++ b/README.md @@ -817,6 +817,8 @@ to use the development version for `pipelines`. - [x] Copy local composer cache into container for better (offline) usage in PHP projects (see [Populate Caches](doc/PIPELINES-CACHES.md#populate-caches)) +- [x] Run scripts with `/bin/bash` if available ([#17]) (*bash-runner* + feature) - [ ] Support for `BITBUCKET_DOCKER_HOST_INTERNAL` environment variable / host.docker.internal hostname within pipelines - [ ] Count `BITBUCKET_BUILD_NUMBER` on a per project basis (*build-number* @@ -857,6 +859,7 @@ to use the development version for `pipelines`. - [ ] Override the default image name (`--default-image `; never needed this for local run) +[#17]: https://github.com/ktomk/pipelines/issues/17 [#13]: https://github.com/ktomk/pipelines/issues/13 ## References diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 43602c12..41168cb3 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -355,6 +355,14 @@ pipelines: mkdir -p /dev/null && test echo "despite mkdir non-zero exit status ($?), this script part will be fine" - echo "ok." + bash-runner: + - step: + name: "bash runner" + image: ktomk/pipelines:ubuntu-bash + script: + - echo "$HELLO" + - source /etc/os-release + - echo "$PRETTY_NAME" definitions: caches: diff --git a/doc/CONFIGURATION-PARAMETERS.md b/doc/CONFIGURATION-PARAMETERS.md index 9fb3c009..8ac46bf8 100644 --- a/doc/CONFIGURATION-PARAMETERS.md +++ b/doc/CONFIGURATION-PARAMETERS.md @@ -25,6 +25,16 @@ host and within mount binds. [DCK_HOST]: https://docs.docker.com/engine/reference/commandline/cli/#environment-variables +## `script.bash-runner` + +Conditionally run scripts with `/bin/bash` instead of `script.runner` (`/bin/sh` +by default) if `/bin/bash` is a standard file and executable in the container. + +Setting `script.bash-runner` to `false` prevents this check. + +* Default: `true` (bool) +* Related configuration parameter: [`script.runner`](#scriptrunner) + ## `script.exit-early` Executing the step and after scripts use `set -e` to exit @@ -37,6 +47,21 @@ after *any* shell pipe with a non-zero exit status. * Default: `false` (bool). +## `script.runner` + +The default script runner (`/bin/sh`), step scripts are run with this command. + +The step script is passed to the runner via standard input. The runner must be +executable in the container. + +When `script.bash-runner` is `true` (default), the `script.runner` may be +overridden by `/bin/bash` if found in the container. This behaviour can be +disabled by setting `script.bash-runner` is `false`. + +* Default: `/bin/sh` +* Related configuration parameter: [`script.bash-runner`](#scriptbash-runner) +* Related option: `--step-script` + ## `step.clone-path` Mount point / destination of the project files within a pipeline step diff --git a/lib/container/ubuntu-bash.sh b/lib/container/ubuntu-bash.sh new file mode 100755 index 00000000..a2bec59e --- /dev/null +++ b/lib/container/ubuntu-bash.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# this file is part of pipelines +# +# pipelines:ubuntu-bash docker image +# +# usage: ./ubuntu-bash.sh [] +# +# "docker" by default +# +# requirements: docker (or equivalent) +# +# rationale: +# +# bitbucket pipelines runs the scripts with /bin/bash if available, so +# a user asked for it and we need it to integrate with bash. +# +# +# +set -euo pipefail +IFS=$' \n\t' + +docker_cmd="${1-docker}" +from="docker.io/ubuntu" +tag="docker.io/ktomk/pipelines:ubuntu-bash" + +echo "build '${tag}' from '${from}' with ${docker_cmd}..." + +<<'DOCKERFILE' "${docker_cmd}" build --build-arg FROM="${from}" -t "${tag}" - | sed -e 's/^/ build: /' +ARG FROM +FROM $FROM +RUN echo "HELLO=YOU" > ~/.bashrc +SHELL ["/bin/bash", "-c"] +DOCKERFILE + +echo "test '${tag}' ..." + +# shellcheck disable=SC2016 +"${docker_cmd}" run --rm "${tag}" /bin/bash -c ' +set -x +echo "\$HELLO.......: $HELLO" +echo "\$SHELL.......: $SHELL" +echo "\$BASH_VERSION: $BASH_VERSION" +/bin/bash --version +which bash +ls -lhi "$(which bash)" +ls -lhi /bin/bash +source /etc/os-release +echo "$PRETTY_NAME" +' 2>&1 | sed -e 's/^/ test.: /' + +echo "push '${tag}' ..." + +"${docker_cmd}" push "${tag}" | sed -e 's/^/ push.: /' + +echo "done." diff --git a/src/Runner/StepScriptRunner.php b/src/Runner/StepScriptRunner.php index 53121ec5..8e519b52 100644 --- a/src/Runner/StepScriptRunner.php +++ b/src/Runner/StepScriptRunner.php @@ -73,7 +73,15 @@ public function runStepScript(Step $step) $this->runner->getRunOpts()->getBoolOption('script.exit-early') ); - $status = $this->execScript($buffer, $exec, $name); + $scriptRunner = $this->runner->getRunOpts()->getOption('script.runner'); + if ($this->runner->getRunOpts()->getBoolOption('script.bash-runner')) { + $bashRunner = '/bin/bash'; + 0 === $exec->capture('docker', array( + 'exec', $name, '/bin/sh', '-c', "test -f ${bashRunner} && test -x ${bashRunner}", + )) && $scriptRunner = $bashRunner; + } + + $status = $this->execScript($buffer, $exec, $name, $scriptRunner); if (0 !== $status) { $streams->err(sprintf("script non-zero exit status: %d\n", $status)); } @@ -86,7 +94,7 @@ public function runStepScript(Step $step) $buffer = StepScriptWriter::writeAfterScript($script, $status); - $afterStatus = $this->execScript($buffer, $exec, $name); + $afterStatus = $this->execScript($buffer, $exec, $name, $scriptRunner); if (0 !== $afterStatus) { $streams->err(sprintf("after-script non-zero exit status: %d\n", $afterStatus)); } @@ -99,13 +107,14 @@ public function runStepScript(Step $step) * @param string $script "\n" terminated script lines * @param Exec $exec * @param string $name + * @param string $runner executable (absolute path), defaults to "/bin/sh" * * @return int */ - private function execScript($script, Exec $exec, $name) + private function execScript($script, Exec $exec, $name, $runner = '/bin/sh') { $buffer = Lib::cmd("<<'SCRIPT' docker", array( - 'exec', '-i', $name, '/bin/sh', + 'exec', '-i', $name, $runner, )) . "\n"; $buffer .= $script; $buffer .= "SCRIPT\n"; diff --git a/src/Runner/StepScriptWriter.php b/src/Runner/StepScriptWriter.php index 9b28de14..f4038031 100644 --- a/src/Runner/StepScriptWriter.php +++ b/src/Runner/StepScriptWriter.php @@ -43,6 +43,7 @@ public static function writeStepScript(array $script, $scriptExitEarly = false) { $buffer = "# this /bin/sh script is generated from a pipeline script\n"; $buffer .= "set -e\n"; + $buffer .= 'test "$0" = "/bin/bash" && if [ -f ~/.bashrc ]; then source ~/.bashrc; fi' . "\n"; $buffer .= self::generateScriptBody( $script, self::getLinePostCommand($scriptExitEarly) diff --git a/src/Utility/ConfigOptions.php b/src/Utility/ConfigOptions.php index e92ba165..4fc6a7c8 100644 --- a/src/Utility/ConfigOptions.php +++ b/src/Utility/ConfigOptions.php @@ -75,6 +75,7 @@ public function run() public function parse(Args $args) { $options = $this->options; + $booleans = array('true' => true, 'false' => false); while (null !== $setting = $args->getOptionArgument('c')) { list($name, $value) = explode('=', $setting, 2) + array('undef', null); @@ -85,6 +86,7 @@ public function parse(Args $args) throw new \InvalidArgumentException(sprintf('not a %s: "%s"', $name, $value)); } $type = null === $default ? 'string' : gettype($default); + 'boolean' === $type && $value = isset($booleans[$value]) ? $booleans[$value] : $value; settype($value, $type) && $options->definition[$name] = array($value); } } diff --git a/src/Utility/Options.php b/src/Utility/Options.php index 31828fdc..5d8d74e6 100644 --- a/src/Utility/Options.php +++ b/src/Utility/Options.php @@ -29,6 +29,8 @@ public static function create() $definition = array( 'docker.socket.path' => array('/var/run/docker.sock', null), 'docker.client.path' => array('/usr/bin/docker', null), + 'script.runner' => array('/bin/sh', null), + 'script.bash-runner' => array(true, null), 'script.exit-early' => array(false, null), 'step.clone-path' => array('/app', Types::ABSPATH), ); diff --git a/test/integration/Runner/Docker/Binary/RepositoryTest.php b/test/integration/Runner/Docker/Binary/RepositoryTest.php index 46a1b621..c13846a4 100644 --- a/test/integration/Runner/Docker/Binary/RepositoryTest.php +++ b/test/integration/Runner/Docker/Binary/RepositoryTest.php @@ -132,6 +132,7 @@ public function testRunnerWithDeploy() $exec->expect('capture', 'docker', 0, 'container id by name'); $exec->expect('capture', 'docker', 0, 'run container'); + $exec->expect('capture', 'docker', 1, 'test for /bin/bash'); $exec->expect('pass', '~^<<\'SCRIPT\' docker exec ~', 0, 'run step script'); $exec->expect('capture', 'docker', 'kill'); $exec->expect('capture', 'docker', 'rm'); diff --git a/test/unit/Runner/StepRunnerTest.php b/test/unit/Runner/StepRunnerTest.php index e5903634..405790e2 100644 --- a/test/unit/Runner/StepRunnerTest.php +++ b/test/unit/Runner/StepRunnerTest.php @@ -97,6 +97,7 @@ public function testCopy() ->expect('capture', 'docker', 0, 'run step container') ->expect('pass', $this->deploy_copy_cmd, 0, 'copy deployment /app create') ->expect('pass', $this->deploy_copy_cmd_2, 0, 'copy deployment /app copy') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 0, 'run step script') ->expect('capture', 'docker', 0, 'docker kill') ->expect('capture', 'docker', 0, 'docker rm'); @@ -161,6 +162,7 @@ public function testKeepContainerOnErrorWithNonExistentContainer() $exec ->expect('capture', 'docker', 1, 'no id for name of potential re-use') ->expect('capture', 'docker', 0, 'run the container') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 255); $this->keepContainerOnErrorExecTest($exec); @@ -176,6 +178,7 @@ public function testKeepContainerOnErrorWithExistingContainer() $exec = new ExecTester($this); $exec ->expect('capture', 'docker', $containerId) # id for name of potential re-use + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 255); $this->keepContainerOnErrorExecTest($exec, $containerId); @@ -191,6 +194,7 @@ public function testZapExistingContainer() ->expect('capture', 'docker', 0, 'docker run step container') ->expect('pass', $this->deploy_copy_cmd, 0, 'deploy copy stage 1') ->expect('pass', $this->deploy_copy_cmd_2, 0, 'deploy copy stage 1') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 0) ->expect('capture', 'docker', 0, 'docker kill') ->expect('capture', 'docker', 0, 'docker rm'); @@ -210,6 +214,7 @@ public function testKeepExistingContainer() $exec = new ExecTester($this); $exec ->expect('capture', 'docker', "123456789\n", 'existing id') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 0); $runner = $this->createTestStepRunner($exec, (Flags::FLAG_DOCKER_KILL | Flags::FLAG_DOCKER_REMOVE) ^ Flags::FLAGS); @@ -342,6 +347,7 @@ public function testRunStepWithPipDockerSocket() $exec->expect('capture', 'docker', $buffer, 'obtain socket bind'); $exec->expect('capture', 'docker', 0, 'obtain mount bind'); $exec->expect('capture', 'docker', 0, 'run'); + $exec->expect('capture', 'docker', 1, 'test for /bin/bash'); $exec->expect('pass', '~ docker exec ~', 0, 'script'); $exec->expect('capture', 'docker', 0, 'kill'); $exec->expect('capture', 'docker', 0, 'rm'); @@ -364,6 +370,7 @@ public function testDockerClientInjection() $exec ->expect('capture', 'docker', 1, 'no id for name of potential re-use') ->expect('capture', 'docker', 0, 'run the container') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 0, 'run step script') ->expect('capture', 'docker', 0, 'kill') ->expect('capture', 'docker', 0, 'rm'); @@ -431,6 +438,7 @@ public function testDockerClientMount() $exec->expect('capture', 'docker', $buffer, 'obtain client bind'); $exec->expect('capture', 'docker', 0, 'obtain mount bind'); $exec->expect('capture', 'docker', 0, 'run'); + $exec->expect('capture', 'docker', 1, 'test for /bin/bash'); $exec->expect('pass', '~ docker exec ~', 0, 'script'); $exec->expect('capture', 'docker', 0, 'kill'); $exec->expect('capture', 'docker', 0, 'rm'); @@ -483,6 +491,7 @@ public function testArtifacts() ->expect('capture', 'docker', 0, 'docker run step container') ->expect('pass', $this->deploy_copy_cmd, 0) ->expect('pass', $this->deploy_copy_cmd_2, 0) + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 0) ->expect('capture', 'docker', './build/foo-package.tgz') ->expect('pass', 'docker exec -w /app \'*dry-run*\' tar c -f - build/foo-package.tgz | tar x -f - -C ' . $tmpProjectDir, 0) @@ -504,6 +513,7 @@ public function testArtifactsNoMatch() ->expect('capture', 'docker', 0, 'docker run step container') ->expect('pass', $this->deploy_copy_cmd, 0) ->expect('pass', $this->deploy_copy_cmd_2, 0) + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 0) ->expect('capture', 'docker', './build/foo-package.tgz') ->expect('capture', 'docker', 0) # docker kill @@ -527,6 +537,7 @@ public function testArtifactsFailure() ->expect('capture', 'docker', 0, 'docker run step container') ->expect('pass', $this->deploy_copy_cmd, 0) ->expect('pass', $this->deploy_copy_cmd_2, 0) + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~ docker exec ~', 0) ->expect('capture', 'docker', './build/foo-package.tgz') ->expect( @@ -571,6 +582,7 @@ public function testAfterScript() $exec ->expect('capture', 'docker', 1, 'zap') ->expect('capture', 'docker', 0, 'docker run step container') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~<<\'SCRIPT\' docker exec ~', 0, 'script') ->expect('pass', '~<<\'SCRIPT\' docker exec ~', 0, 'after-script') ->expect('capture', 'docker', 0, 'docker kill') @@ -591,6 +603,7 @@ public function testAfterScriptFailing() $exec ->expect('capture', 'docker', 1, 'zap') ->expect('capture', 'docker', 0, 'docker run step container') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~<<\'SCRIPT\' docker exec ~', 0, 'script') ->expect('pass', '~<<\'SCRIPT\' docker exec ~', 123, 'after-script') ->expect('capture', 'docker', 0, 'docker kill') @@ -616,6 +629,7 @@ public function testServicesObtainNetworkAndShutdown() ->expect('capture', 'docker', 1, 'zap') ->expect('capture', 'docker', 0, 'docker run step container') ->expect('capture', 'docker', 0, 'run services') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~<<\'SCRIPT\' docker exec ~', 0, 'script') ->expect('capture', 'docker', 0, 'docker kill') ->expect('capture', 'docker', 0, 'docker rm') @@ -641,6 +655,7 @@ public function testCaches() $exec ->expect('capture', 'docker', 1, 'zap') ->expect('capture', 'docker', 0, 'docker run step container') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~<<\'SCRIPT\' docker exec ~', 0, 'script') ->expect('capture', '~^docker exec ~', 0, 'caches: map path') ->expect('capture', '~>.*/composer\.tar docker cp~', 0, 'caches: copy out') @@ -676,6 +691,7 @@ public function testCachesPrePopulated() ->expect('capture', '~^docker exec ~', 0, 'caches: map path') ->expect('capture', '~^docker exec .* mkdir -p ~', 0, 'caches: mkdir in container') ->expect('capture', '~<.*/composer\.tar docker cp ~', 0, 'caches: copy in') + ->expect('capture', 'docker', 1, 'test for /bin/bash') ->expect('pass', '~<<\'SCRIPT\' docker exec ~', 0, 'script') ->expect('capture', '~^docker exec ~', 0, 'caches: map path') ->expect('capture', '~>.*/composer\.tar docker cp~', 0, 'caches: copy out') diff --git a/test/unit/Runner/StepScriptRunnerTest.php b/test/unit/Runner/StepScriptRunnerTest.php index 3c8928aa..71bbbdcd 100644 --- a/test/unit/Runner/StepScriptRunnerTest.php +++ b/test/unit/Runner/StepScriptRunnerTest.php @@ -40,6 +40,20 @@ public function testRunStepScript() $scriptRunner = new StepScriptRunner($runner, '*test-run*'); $step = $this->createTestStepFromFixture('steps.yml'); + $exec->expect('capture', 'docker', 1, 'test for /bin/bash'); + $exec->expect('pass', '~^<<\'SCRIPT\' ~'); + + $actual = $scriptRunner->runStepScript($step); + self::assertSame(0, $actual); + } + + public function testRunStepScriptWithBinBash() + { + $runner = $this->mockRunner($exec); + $scriptRunner = new StepScriptRunner($runner, '*test-run*'); + + $step = $this->createTestStepFromFixture('steps.yml'); + $exec->expect('capture', 'docker', 0, 'test for /bin/bash'); $exec->expect('pass', '~^<<\'SCRIPT\' ~'); $actual = $scriptRunner->runStepScript($step); @@ -53,6 +67,7 @@ public function testRunStepScriptAndAfterScript() $step = $this->createTestStepFromFixture('after-script.yml'); + $exec->expect('capture', 'docker', 1, 'test for /bin/bash'); $exec->expect('pass', '~^<<\'SCRIPT\' ~', 3); $exec->expect('pass', '~^<<\'SCRIPT\' ~', 1); @@ -72,6 +87,7 @@ public function testRunStepScriptWithPipe() $step = $this->createTestStepFromFixture('pipe.yml', 1, 'branches/develop'); + $exec->expect('capture', 'docker', 1, 'test for /bin/bash'); $exec->expect('pass', '~^<<\'SCRIPT\' ~'); $actual = $scriptRunner->runStepScript($step); @@ -88,6 +104,7 @@ public function testRunStepScriptWithAfterScriptPipe() $this->expectOutputString('After script:' . "\n"); + $exec->expect('capture', 'docker', 1, 'test for /bin/bash'); $exec->expect('pass', '~^<<\'SCRIPT\' ~', 0, 'script'); $exec->expect('pass', '~^<<\'SCRIPT\' ~', 0, 'after-script');