From 1c486b2c73cb2ae896dd77e0f0ec060a47f15cd7 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:10:07 -0700 Subject: [PATCH 001/105] Add one codecov --- codecov.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..7a4789e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,11 @@ +ignore: + - "libsqlite3-sys/bindgen-bindings" + - "libsqlite3-sys/sqlite3" +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true From 1b8c3056e6a015949896ca20815719930ec48051 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:21:57 -0700 Subject: [PATCH 002/105] Merge another codecov --- codecov.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/codecov.yml b/codecov.yml index 7a4789e..7b19db2 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,11 +1,17 @@ -ignore: - - "libsqlite3-sys/bindgen-bindings" - - "libsqlite3-sys/sqlite3" coverage: + range: 70..100 + round: down + precision: 2 status: project: default: - informational: true - patch: - default: - informational: true + threshold: 2% + +# Tests aren't important for coverage +ignore: + - "tests" + +# Make less noisy comments +comment: + layout: "files" + require_changes: yes From 7f34f791c0a5c3f2c2ce2ed7e43ff12ed123c62c Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:22:18 -0700 Subject: [PATCH 003/105] Merge another codecov --- codecov.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index 7b19db2..7864a9e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,13 +1,14 @@ +# Hold ourselves to a high bar coverage: - range: 70..100 + range: 85..100 round: down precision: 2 status: project: default: - threshold: 2% + threshold: 1% -# Tests aren't important for coverage +# Test files aren't important for coverage ignore: - "tests" From 510b69615dd47cb63584976512cb95b265cb22bf Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:23:03 -0700 Subject: [PATCH 004/105] Merge another codecov --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 7864a9e..1e0fb06 100644 --- a/codecov.yml +++ b/codecov.yml @@ -2,7 +2,7 @@ coverage: range: 85..100 round: down - precision: 2 + precision: 1 status: project: default: From b32648cabb5862b0814ab0abd6d5c81498758270 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:24:02 -0700 Subject: [PATCH 005/105] Merge another codecov --- codecov.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/codecov.yml b/codecov.yml index 1e0fb06..ff4f571 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,18 +1,21 @@ -# Hold ourselves to a high bar +# ref: https://docs.codecov.com/docs/codecovyml-reference coverage: + # Hold ourselves to a high bar range: 85..100 round: down precision: 1 status: + # ref: https://docs.codecov.com/docs/commit-status project: default: + # Avoid false negatives threshold: 1% # Test files aren't important for coverage ignore: - "tests" -# Make less noisy comments +# Make comments less noisy comment: layout: "files" require_changes: yes From 126c9a3a35d5ac428c22883d36f0aac69d2e20e9 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:25:23 -0700 Subject: [PATCH 006/105] Place codecov config under .github --- codecov.yml => github/codecov.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename codecov.yml => github/codecov.yml (100%) diff --git a/codecov.yml b/github/codecov.yml similarity index 100% rename from codecov.yml rename to github/codecov.yml From 11027d3f75ced20536b99225edccf34f286dd4e0 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:27:27 -0700 Subject: [PATCH 007/105] Add (only) ASAN workflow --- github/workflows/asan.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 github/workflows/asan.yml diff --git a/github/workflows/asan.yml b/github/workflows/asan.yml new file mode 100644 index 0000000..08afae9 --- /dev/null +++ b/github/workflows/asan.yml @@ -0,0 +1,24 @@ +on: + push: + branches: [main] + pull_request: +name: Address sanitizer +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + - uses: actions/checkout@v2 + - name: cargo test -Zsanitizer=address + uses: actions-rs/cargo@v1 + with: + command: test + # only --lib --tests b/c of https://github.com/rust-lang/rust/issues/53945 + args: --lib --tests --all-features --target x86_64-unknown-linux-gnu + env: + ASAN_OPTIONS: "detect_odr_violation=0:detect_leaks=0" + RUSTFLAGS: "-Z sanitizer=address" From bbdbd96ec709e3cc83a081cf821fdfffce85ecb5 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:27:38 -0700 Subject: [PATCH 008/105] Add first coverage workflow --- github/workflows/coverage.yml | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 github/workflows/coverage.yml diff --git a/github/workflows/coverage.yml b/github/workflows/coverage.yml new file mode 100644 index 0000000..6effa8c --- /dev/null +++ b/github/workflows/coverage.yml @@ -0,0 +1,66 @@ +on: + push: + branches: [master] + paths-ignore: + - 'build_doc.sh' + - 'check.sh' + - 'run_ci_tests.sh' + - 'start_sshd.sh' + - 'stop_sshd.sh' + pull_request: + paths-ignore: + - 'build_doc.sh' + - 'check.sh' + - 'run_ci_tests.sh' + - 'start_sshd.sh' + - 'stop_sshd.sh' +name: coverage +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + components: llvm-tools-preview + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - run: | + # Wait for startup of openssh-server + timeout 15 ./wait_for_sshd_start_up.sh + chmod 600 .test-key + mkdir /tmp/openssh-rs + ssh -i .test-key -v -p 2222 -l test-user localhost -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/tmp/openssh-rs/known_hosts whoami + name: Test ssh connectivity + - run: | + eval $(ssh-agent) + echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV + echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> $GITHUB_ENV + cat .test-key | ssh-add - + name: Set up ssh-agent + - name: Generate code coverage + run: cargo llvm-cov --all-features --lcov --output-path lcov.info + env: + # makes all the ignored tests not ignored + RUSTFLAGS: --cfg=ci + # we cannot use 127.0.0.1 (the default here) + # since we are running from a different container + TEST_HOST: ssh://test-user@localhost:2222 + XDG_RUNTIME_DIR: /tmp + - name: Upload to codecov.io + uses: codecov/codecov-action@v2 + with: + fail_ci_if_error: true + services: + openssh: + image: linuxserver/openssh-server:amd64-latest + ports: + - 2222:2222 + env: + USER_NAME: test-user + PUBLIC_KEY: |- + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGzHvK2pKtSlZXP9tPYOOBb/xn0IiC9iLMS355AYUPC7 + DOCKER_MODS: linuxserver/mods:openssh-server-ssh-tunnel From fe6ba380bd39c665e9d9a2153b2dc5287fe25cae Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:29:55 -0700 Subject: [PATCH 009/105] Merge another coverage.yml --- github/workflows/coverage.yml | 42 ----------------------------------- 1 file changed, 42 deletions(-) diff --git a/github/workflows/coverage.yml b/github/workflows/coverage.yml index 6effa8c..f07bde1 100644 --- a/github/workflows/coverage.yml +++ b/github/workflows/coverage.yml @@ -1,19 +1,7 @@ on: push: branches: [master] - paths-ignore: - - 'build_doc.sh' - - 'check.sh' - - 'run_ci_tests.sh' - - 'start_sshd.sh' - - 'stop_sshd.sh' pull_request: - paths-ignore: - - 'build_doc.sh' - - 'check.sh' - - 'run_ci_tests.sh' - - 'start_sshd.sh' - - 'stop_sshd.sh' name: coverage jobs: test: @@ -28,39 +16,9 @@ jobs: components: llvm-tools-preview - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - - run: | - # Wait for startup of openssh-server - timeout 15 ./wait_for_sshd_start_up.sh - chmod 600 .test-key - mkdir /tmp/openssh-rs - ssh -i .test-key -v -p 2222 -l test-user localhost -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/tmp/openssh-rs/known_hosts whoami - name: Test ssh connectivity - - run: | - eval $(ssh-agent) - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV - echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> $GITHUB_ENV - cat .test-key | ssh-add - - name: Set up ssh-agent - name: Generate code coverage run: cargo llvm-cov --all-features --lcov --output-path lcov.info - env: - # makes all the ignored tests not ignored - RUSTFLAGS: --cfg=ci - # we cannot use 127.0.0.1 (the default here) - # since we are running from a different container - TEST_HOST: ssh://test-user@localhost:2222 - XDG_RUNTIME_DIR: /tmp - name: Upload to codecov.io uses: codecov/codecov-action@v2 with: fail_ci_if_error: true - services: - openssh: - image: linuxserver/openssh-server:amd64-latest - ports: - - 2222:2222 - env: - USER_NAME: test-user - PUBLIC_KEY: |- - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGzHvK2pKtSlZXP9tPYOOBb/xn0IiC9iLMS355AYUPC7 - DOCKER_MODS: linuxserver/mods:openssh-server-ssh-tunnel From 1fe2a6d008275efaff56200b8fba5ecc252aa970 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:31:17 -0700 Subject: [PATCH 010/105] Merge another coverage.yml --- github/workflows/coverage.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/github/workflows/coverage.yml b/github/workflows/coverage.yml index f07bde1..677ad58 100644 --- a/github/workflows/coverage.yml +++ b/github/workflows/coverage.yml @@ -8,6 +8,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + submodules: true - uses: actions-rs/toolchain@v1 with: profile: minimal From 225ad3978688c093f4670ec04352d465076f39d3 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:31:58 -0700 Subject: [PATCH 011/105] Add first features workflow --- github/workflows/features.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 github/workflows/features.yml diff --git a/github/workflows/features.yml b/github/workflows/features.yml new file mode 100644 index 0000000..2d8c1fe --- /dev/null +++ b/github/workflows/features.yml @@ -0,0 +1,27 @@ +on: + push: + branches: [master] + pull_request: +name: cargo hack +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: actions/checkout@v2 + with: + submodules: true + - name: Install cargo-hack + uses: actions-rs/install@v0.1 + with: + crate: cargo-hack + version: latest + use-tool-cache: true + - name: cargo hack + uses: actions-rs/cargo@v1 + with: + command: hack + args: --feature-powerset --exclude-no-default-features check From 922692a2977a4c93786a0ecbe11fc01501361aad Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:32:32 -0700 Subject: [PATCH 012/105] Merge another features workflow --- github/workflows/features.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github/workflows/features.yml b/github/workflows/features.yml index 2d8c1fe..f6acf15 100644 --- a/github/workflows/features.yml +++ b/github/workflows/features.yml @@ -24,4 +24,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: hack - args: --feature-powerset --exclude-no-default-features check + args: --feature-powerset check From 043eb24611b5272a04082d63566837a9efbc71e9 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:32:58 -0700 Subject: [PATCH 013/105] Merge another features workflow --- github/workflows/features.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github/workflows/features.yml b/github/workflows/features.yml index f6acf15..d9ca4c0 100644 --- a/github/workflows/features.yml +++ b/github/workflows/features.yml @@ -24,4 +24,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: hack - args: --feature-powerset check + args: --feature-powerset check --all-targets From d8c8a99dea99b437eefc56e5b873a863a4446c51 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:33:48 -0700 Subject: [PATCH 014/105] Merge another features workflow --- github/workflows/features.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/github/workflows/features.yml b/github/workflows/features.yml index d9ca4c0..c2c3920 100644 --- a/github/workflows/features.yml +++ b/github/workflows/features.yml @@ -15,11 +15,7 @@ jobs: with: submodules: true - name: Install cargo-hack - uses: actions-rs/install@v0.1 - with: - crate: cargo-hack - version: latest - use-tool-cache: true + uses: taiki-e/install-action@cargo-hack - name: cargo hack uses: actions-rs/cargo@v1 with: From f67cad0f915deebcdf7ceb89ffdd0925bc910153 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:34:34 -0700 Subject: [PATCH 015/105] Add (only) loom workflow --- github/workflows/loom.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 github/workflows/loom.yml diff --git a/github/workflows/loom.yml b/github/workflows/loom.yml new file mode 100644 index 0000000..9246b40 --- /dev/null +++ b/github/workflows/loom.yml @@ -0,0 +1,22 @@ +on: + push: + branches: [main] + pull_request: +name: Loom +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + - uses: actions/checkout@v2 + - name: cargo test --test loom + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --test loom + env: + LOOM_MAX_PREEMPTIONS: 2 + RUSTFLAGS: "--cfg loom" From bf66d94f15b7288f417cfae0eab6542e2e100daf Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:34:43 -0700 Subject: [PATCH 016/105] Add (only) LSAN workflow --- github/workflows/lsan.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 github/workflows/lsan.yml diff --git a/github/workflows/lsan.yml b/github/workflows/lsan.yml new file mode 100644 index 0000000..69ce0df --- /dev/null +++ b/github/workflows/lsan.yml @@ -0,0 +1,32 @@ +on: + push: + branches: [main] + pull_request: +name: Leak sanitizer +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + - uses: actions/checkout@v2 + - run: | + # to get the symbolizer for debug symbol resolution + sudo apt install llvm + # to fix buggy leak analyzer: + # https://github.com/japaric/rust-san#unrealiable-leaksanitizer + sed -i '/\[features\]/i [profile.dev]' Cargo.toml + sed -i '/profile.dev/a opt-level = 1' Cargo.toml + cat Cargo.toml + name: Enable debug symbols + - name: cargo test -Zsanitizer=leak + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-features --target x86_64-unknown-linux-gnu + env: + RUSTFLAGS: "-Z sanitizer=leak" + LSAN_OPTIONS: "suppressions=lsan-suppressions.txt" From c65a7c4f87be9ddea9e34eb254f3b6d5933db4ef Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:35:09 -0700 Subject: [PATCH 017/105] Add first minial workflow --- github/workflows/minimal.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 github/workflows/minimal.yml diff --git a/github/workflows/minimal.yml b/github/workflows/minimal.yml new file mode 100644 index 0000000..b4cf571 --- /dev/null +++ b/github/workflows/minimal.yml @@ -0,0 +1,31 @@ +on: + push: + branches: [master] + pull_request: +name: With dependencies at minimal versions +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: actions/checkout@v2 + with: + submodules: true + - name: cargo update -Zminimal-versions + uses: actions-rs/cargo@v1 + with: + command: update + toolchain: nightly + args: -Zminimal-versions + - name: cargo test + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-features --all-targets From 77079d77cb4aa288bda667917667cfaee87bd361 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:36:29 -0700 Subject: [PATCH 018/105] Add (only) miri workflow --- github/workflows/miri.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 github/workflows/miri.yml diff --git a/github/workflows/miri.yml b/github/workflows/miri.yml new file mode 100644 index 0000000..e79ab42 --- /dev/null +++ b/github/workflows/miri.yml @@ -0,0 +1,25 @@ +on: + push: + branches: [main] + pull_request: +name: Miri +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: | + echo "NIGHTLY=nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/miri)" >> $GITHUB_ENV + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ env.NIGHTLY }} + override: true + components: miri + - uses: actions/checkout@v2 + - name: cargo miri test + uses: actions-rs/cargo@v1 + with: + command: miri + args: test + env: + MIRIFLAGS: "-Zmiri-tag-raw-pointers" From 9b48ae326374d8d8609a65649026fa09f8a68c7f Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:36:43 -0700 Subject: [PATCH 019/105] Add first msrv workflow --- github/workflows/msrv.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 github/workflows/msrv.yml diff --git a/github/workflows/msrv.yml b/github/workflows/msrv.yml new file mode 100644 index 0000000..aebe1f8 --- /dev/null +++ b/github/workflows/msrv.yml @@ -0,0 +1,19 @@ +on: + push: + branches: [master] + pull_request: +name: Minimum Supported Rust Version +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.48.0 # nom 7 + override: true + - uses: actions/checkout@v2 + - name: cargo +1.48.0 check + uses: actions-rs/cargo@v1 + with: + command: check From b60aa5589ac569446a5128453983dee9bb504666 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:37:03 -0700 Subject: [PATCH 020/105] Merge another msrv workflow --- github/workflows/msrv.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github/workflows/msrv.yml b/github/workflows/msrv.yml index aebe1f8..ee1b2bf 100644 --- a/github/workflows/msrv.yml +++ b/github/workflows/msrv.yml @@ -10,10 +10,10 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: 1.48.0 # nom 7 + toolchain: 1.56.0 # 2021 edition requires 1.56 override: true - uses: actions/checkout@v2 - - name: cargo +1.48.0 check + - name: cargo +1.56.0 check uses: actions-rs/cargo@v1 with: command: check From 1113c895d862ce860c82596cac973ad075ef1ac6 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:37:31 -0700 Subject: [PATCH 021/105] Merge another msrv workflow --- github/workflows/msrv.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github/workflows/msrv.yml b/github/workflows/msrv.yml index ee1b2bf..4a3a92e 100644 --- a/github/workflows/msrv.yml +++ b/github/workflows/msrv.yml @@ -10,10 +10,10 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: 1.56.0 # 2021 edition requires 1.56 + toolchain: 1.56.1 # 2021 edition requires 1.56 override: true - uses: actions/checkout@v2 - - name: cargo +1.56.0 check + - name: cargo +1.56.1 check uses: actions-rs/cargo@v1 with: command: check From e6ef8e3166b93c22af938872a547e104f2601587 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:38:01 -0700 Subject: [PATCH 022/105] Merge another msrv workflow --- github/workflows/msrv.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/github/workflows/msrv.yml b/github/workflows/msrv.yml index 4a3a92e..ac0dc41 100644 --- a/github/workflows/msrv.yml +++ b/github/workflows/msrv.yml @@ -13,6 +13,8 @@ jobs: toolchain: 1.56.1 # 2021 edition requires 1.56 override: true - uses: actions/checkout@v2 + with: + submodules: true - name: cargo +1.56.1 check uses: actions-rs/cargo@v1 with: From c74ee968a1aafec9e839dee907f0137e6356feff Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:38:56 -0700 Subject: [PATCH 023/105] Add (only) no-std workflow --- github/workflows/nostd.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 github/workflows/nostd.yml diff --git a/github/workflows/nostd.yml b/github/workflows/nostd.yml new file mode 100644 index 0000000..433e11d --- /dev/null +++ b/github/workflows/nostd.yml @@ -0,0 +1,24 @@ +on: + push: + branches: [main] + pull_request: +name: no-std +jobs: + nostd: + runs-on: ubuntu-latest + name: ${{ matrix.target }} + strategy: + matrix: + target: [thumbv7m-none-eabi, aarch64-unknown-none] + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + target: ${{ matrix.target }} + - uses: actions/checkout@v2 + - name: cargo check + uses: actions-rs/cargo@v1 + with: + command: check + args: --target ${{ matrix.target }} --no-default-features From 92379c862376607f7caca04e470c09671922f238 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:39:09 -0700 Subject: [PATCH 024/105] Add first os-check workflow --- github/workflows/os-check.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 github/workflows/os-check.yml diff --git a/github/workflows/os-check.yml b/github/workflows/os-check.yml new file mode 100644 index 0000000..bbcb475 --- /dev/null +++ b/github/workflows/os-check.yml @@ -0,0 +1,24 @@ +on: + push: + branches: [master] + pull_request: +name: cargo check +jobs: + os-check: + runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: actions/checkout@v2 + - name: cargo check + uses: actions-rs/cargo@v1 + with: + command: check + args: --all-features --all-targets From 05dd4680bf90603c70cb7cd406299675441fe59d Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:39:59 -0700 Subject: [PATCH 025/105] Merge another os-check workflow --- github/workflows/os-check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/github/workflows/os-check.yml b/github/workflows/os-check.yml index bbcb475..285499b 100644 --- a/github/workflows/os-check.yml +++ b/github/workflows/os-check.yml @@ -2,7 +2,7 @@ on: push: branches: [master] pull_request: -name: cargo check +name: os check jobs: os-check: runs-on: ${{ matrix.os }} @@ -17,8 +17,8 @@ jobs: profile: minimal toolchain: stable - uses: actions/checkout@v2 - - name: cargo check + - name: cargo test uses: actions-rs/cargo@v1 with: - command: check + command: test args: --all-features --all-targets From bc3f55118617b5ffe1ea479c4f6d7d2167b86d36 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:40:29 -0700 Subject: [PATCH 026/105] Add first style workflow --- github/workflows/style.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 github/workflows/style.yml diff --git a/github/workflows/style.yml b/github/workflows/style.yml new file mode 100644 index 0000000..74a877c --- /dev/null +++ b/github/workflows/style.yml @@ -0,0 +1,36 @@ +on: + push: + branches: [main] + pull_request: +name: lint +jobs: + style: + runs-on: ubuntu-latest + name: ${{ matrix.toolchain }} + strategy: + fail-fast: false + matrix: + toolchain: [stable, beta] + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.toolchain }} + components: rustfmt, clippy + - uses: actions/checkout@v2 + - name: cargo fmt --check + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --check + - name: cargo doc + uses: actions-rs/cargo@v1 + if: always() + with: + command: doc + args: --no-deps --all-features + - name: cargo clippy + uses: actions-rs/clippy-check@v1 + if: always() + with: + token: ${{ secrets.GITHUB_TOKEN }} From fe460400ed2259af7e17f5ff51742137623e9e8e Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:41:00 -0700 Subject: [PATCH 027/105] Merge another style workflow --- github/workflows/style.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/github/workflows/style.yml b/github/workflows/style.yml index 74a877c..d3320bc 100644 --- a/github/workflows/style.yml +++ b/github/workflows/style.yml @@ -23,14 +23,24 @@ jobs: with: command: fmt args: --check - - name: cargo doc - uses: actions-rs/cargo@v1 - if: always() - with: - command: doc - args: --no-deps --all-features - name: cargo clippy uses: actions-rs/clippy-check@v1 if: always() with: token: ${{ secrets.GITHUB_TOKEN }} + doc: + runs-on: ubuntu-latest + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + - uses: actions/checkout@v2 + - name: cargo doc + uses: actions-rs/cargo@v1 + with: + toolchain: nightly + command: doc + args: --no-deps --all-features + env: + RUSTDOCFLAGS: --cfg docsrs From 3bd8b12ec08910b2609cdfc843474d5b83ff7dbc Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:41:57 -0700 Subject: [PATCH 028/105] Merge another style workflow --- github/workflows/style.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/github/workflows/style.yml b/github/workflows/style.yml index d3320bc..648b6a1 100644 --- a/github/workflows/style.yml +++ b/github/workflows/style.yml @@ -18,6 +18,8 @@ jobs: toolchain: ${{ matrix.toolchain }} components: rustfmt, clippy - uses: actions/checkout@v2 + with: + submodules: true - name: cargo fmt --check uses: actions-rs/cargo@v1 with: From 8953a88abecc66ea7811766b46aff6a5fd767124 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:42:10 -0700 Subject: [PATCH 029/105] Add first test workflow --- github/workflows/test.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 github/workflows/test.yml diff --git a/github/workflows/test.yml b/github/workflows/test.yml new file mode 100644 index 0000000..a82ac61 --- /dev/null +++ b/github/workflows/test.yml @@ -0,0 +1,29 @@ +on: + push: + branches: [master] + pull_request: +name: cargo test +jobs: + test: + runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} / ${{ matrix.toolchain }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + toolchain: [stable] + include: + - os: ubuntu-latest + toolchain: beta + - os: ubuntu-latest + toolchain: nightly + steps: + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.toolchain }} + - uses: actions/checkout@v2 + - name: cargo test + uses: actions-rs/cargo@v1 + with: + command: test From 0910d977fc68082220d493bef07bc9d5f2265fc7 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:42:25 -0700 Subject: [PATCH 030/105] Merge another test workflow --- github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/github/workflows/test.yml b/github/workflows/test.yml index a82ac61..5dd762c 100644 --- a/github/workflows/test.yml +++ b/github/workflows/test.yml @@ -27,3 +27,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: test + args: --all-features From 971c3fd9eb5f7d80088caaf5647a74b82d40b860 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:42:52 -0700 Subject: [PATCH 031/105] Merge another test workflow --- github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/github/workflows/test.yml b/github/workflows/test.yml index 5dd762c..de5d3ea 100644 --- a/github/workflows/test.yml +++ b/github/workflows/test.yml @@ -23,8 +23,10 @@ jobs: profile: minimal toolchain: ${{ matrix.toolchain }} - uses: actions/checkout@v2 + with: + submodules: true - name: cargo test uses: actions-rs/cargo@v1 with: command: test - args: --all-features + args: --all-features --all-targets From 2de2235ad3803a978e150fca8d38182eb6ed7a9e Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:46:13 -0700 Subject: [PATCH 032/105] Merge another test workflow --- github/workflows/test.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/github/workflows/test.yml b/github/workflows/test.yml index de5d3ea..b247ef6 100644 --- a/github/workflows/test.yml +++ b/github/workflows/test.yml @@ -5,24 +5,17 @@ on: name: cargo test jobs: test: - runs-on: ${{ matrix.os }} - name: ${{ matrix.os }} / ${{ matrix.toolchain }} + runs-on: ubuntu-latest + name: ubuntu / ${{ matrix.toolchain }} strategy: - fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - toolchain: [stable] - include: - - os: ubuntu-latest - toolchain: beta - - os: ubuntu-latest - toolchain: nightly + toolchain: [stable, beta, nightly] steps: - uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: ${{ matrix.toolchain }} - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: cargo test From 0f90a0b77958b3978b6be3997a09ea5cb9b1bd6b Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:47:38 -0700 Subject: [PATCH 033/105] Make everything use checkout@v3 --- github/workflows/asan.yml | 2 +- github/workflows/features.yml | 2 +- github/workflows/loom.yml | 2 +- github/workflows/lsan.yml | 2 +- github/workflows/minimal.yml | 2 +- github/workflows/miri.yml | 2 +- github/workflows/msrv.yml | 2 +- github/workflows/nostd.yml | 2 +- github/workflows/os-check.yml | 2 +- github/workflows/style.yml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/github/workflows/asan.yml b/github/workflows/asan.yml index 08afae9..5604b7e 100644 --- a/github/workflows/asan.yml +++ b/github/workflows/asan.yml @@ -12,7 +12,7 @@ jobs: profile: minimal toolchain: nightly override: true - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: cargo test -Zsanitizer=address uses: actions-rs/cargo@v1 with: diff --git a/github/workflows/features.yml b/github/workflows/features.yml index c2c3920..3e45d18 100644 --- a/github/workflows/features.yml +++ b/github/workflows/features.yml @@ -11,7 +11,7 @@ jobs: with: profile: minimal toolchain: stable - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Install cargo-hack diff --git a/github/workflows/loom.yml b/github/workflows/loom.yml index 9246b40..8cfe45d 100644 --- a/github/workflows/loom.yml +++ b/github/workflows/loom.yml @@ -11,7 +11,7 @@ jobs: with: toolchain: stable profile: minimal - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: cargo test --test loom uses: actions-rs/cargo@v1 with: diff --git a/github/workflows/lsan.yml b/github/workflows/lsan.yml index 69ce0df..a966913 100644 --- a/github/workflows/lsan.yml +++ b/github/workflows/lsan.yml @@ -12,7 +12,7 @@ jobs: profile: minimal toolchain: nightly override: true - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - run: | # to get the symbolizer for debug symbol resolution sudo apt install llvm diff --git a/github/workflows/minimal.yml b/github/workflows/minimal.yml index b4cf571..f3ed6ac 100644 --- a/github/workflows/minimal.yml +++ b/github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: with: profile: minimal toolchain: stable - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: cargo update -Zminimal-versions diff --git a/github/workflows/miri.yml b/github/workflows/miri.yml index e79ab42..ae9539a 100644 --- a/github/workflows/miri.yml +++ b/github/workflows/miri.yml @@ -15,7 +15,7 @@ jobs: toolchain: ${{ env.NIGHTLY }} override: true components: miri - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: cargo miri test uses: actions-rs/cargo@v1 with: diff --git a/github/workflows/msrv.yml b/github/workflows/msrv.yml index ac0dc41..9758e26 100644 --- a/github/workflows/msrv.yml +++ b/github/workflows/msrv.yml @@ -12,7 +12,7 @@ jobs: profile: minimal toolchain: 1.56.1 # 2021 edition requires 1.56 override: true - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: cargo +1.56.1 check diff --git a/github/workflows/nostd.yml b/github/workflows/nostd.yml index 433e11d..409ae73 100644 --- a/github/workflows/nostd.yml +++ b/github/workflows/nostd.yml @@ -16,7 +16,7 @@ jobs: profile: minimal toolchain: stable target: ${{ matrix.target }} - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: cargo check uses: actions-rs/cargo@v1 with: diff --git a/github/workflows/os-check.yml b/github/workflows/os-check.yml index 285499b..5fc3453 100644 --- a/github/workflows/os-check.yml +++ b/github/workflows/os-check.yml @@ -16,7 +16,7 @@ jobs: with: profile: minimal toolchain: stable - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: cargo test uses: actions-rs/cargo@v1 with: diff --git a/github/workflows/style.yml b/github/workflows/style.yml index 648b6a1..7a583f2 100644 --- a/github/workflows/style.yml +++ b/github/workflows/style.yml @@ -17,7 +17,7 @@ jobs: profile: minimal toolchain: ${{ matrix.toolchain }} components: rustfmt, clippy - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: cargo fmt --check @@ -37,7 +37,7 @@ jobs: with: profile: minimal toolchain: nightly - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: cargo doc uses: actions-rs/cargo@v1 with: From 99ddee84ab05f5d5f37ad30a31d18dd7c72050c9 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 12:47:57 -0700 Subject: [PATCH 034/105] Standardize on 'main' as branch name --- github/workflows/coverage.yml | 2 +- github/workflows/features.yml | 2 +- github/workflows/minimal.yml | 2 +- github/workflows/msrv.yml | 2 +- github/workflows/os-check.yml | 2 +- github/workflows/test.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/github/workflows/coverage.yml b/github/workflows/coverage.yml index 677ad58..375f7a3 100644 --- a/github/workflows/coverage.yml +++ b/github/workflows/coverage.yml @@ -1,6 +1,6 @@ on: push: - branches: [master] + branches: [main] pull_request: name: coverage jobs: diff --git a/github/workflows/features.yml b/github/workflows/features.yml index 3e45d18..ac5e18e 100644 --- a/github/workflows/features.yml +++ b/github/workflows/features.yml @@ -1,6 +1,6 @@ on: push: - branches: [master] + branches: [main] pull_request: name: cargo hack jobs: diff --git a/github/workflows/minimal.yml b/github/workflows/minimal.yml index f3ed6ac..0a558a9 100644 --- a/github/workflows/minimal.yml +++ b/github/workflows/minimal.yml @@ -1,6 +1,6 @@ on: push: - branches: [master] + branches: [main] pull_request: name: With dependencies at minimal versions jobs: diff --git a/github/workflows/msrv.yml b/github/workflows/msrv.yml index 9758e26..d6d9046 100644 --- a/github/workflows/msrv.yml +++ b/github/workflows/msrv.yml @@ -1,6 +1,6 @@ on: push: - branches: [master] + branches: [main] pull_request: name: Minimum Supported Rust Version jobs: diff --git a/github/workflows/os-check.yml b/github/workflows/os-check.yml index 5fc3453..c56b699 100644 --- a/github/workflows/os-check.yml +++ b/github/workflows/os-check.yml @@ -1,6 +1,6 @@ on: push: - branches: [master] + branches: [main] pull_request: name: os check jobs: diff --git a/github/workflows/test.yml b/github/workflows/test.yml index b247ef6..dfc1819 100644 --- a/github/workflows/test.yml +++ b/github/workflows/test.yml @@ -1,6 +1,6 @@ on: push: - branches: [master] + branches: [main] pull_request: name: cargo test jobs: From 87365663b1f49c88c2a3642fece0b2a932001355 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 16:19:55 -0700 Subject: [PATCH 035/105] Missed a submodule checkout --- github/workflows/os-check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/github/workflows/os-check.yml b/github/workflows/os-check.yml index c56b699..db3590b 100644 --- a/github/workflows/os-check.yml +++ b/github/workflows/os-check.yml @@ -17,6 +17,8 @@ jobs: profile: minimal toolchain: stable - uses: actions/checkout@v3 + with: + submodules: true - name: cargo test uses: actions-rs/cargo@v1 with: From 4859c128823805015dc164d58316dc5b25a69264 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 17 Sep 2022 18:16:21 -0700 Subject: [PATCH 036/105] Add TODOs from twitter thread --- github/workflows/locked.yml | 2 ++ github/workflows/scheduled-nightly.yml | 1 + 2 files changed, 3 insertions(+) create mode 100644 github/workflows/locked.yml create mode 100644 github/workflows/scheduled-nightly.yml diff --git a/github/workflows/locked.yml b/github/workflows/locked.yml new file mode 100644 index 0000000..c67a317 --- /dev/null +++ b/github/workflows/locked.yml @@ -0,0 +1,2 @@ +# TODO: https://twitter.com/jonhoo/status/1571290371124260865 +# Maybe also: https://twitter.com/alcuadrado/status/1571291687837732873 diff --git a/github/workflows/scheduled-nightly.yml b/github/workflows/scheduled-nightly.yml new file mode 100644 index 0000000..5a86cca --- /dev/null +++ b/github/workflows/scheduled-nightly.yml @@ -0,0 +1 @@ +# TODO: https://twitter.com/mycoliza/status/1571295690063753218 From afa25312c9c6cf8748629bd3a5c054a688785dfc Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 18 Sep 2022 11:29:34 -0700 Subject: [PATCH 037/105] Practice what you preach --- github/workflows/check.yml | 107 +++++++++++++++++++++++++ github/workflows/coverage.yml | 26 ------ github/workflows/features.yml | 23 ------ github/workflows/locked.yml | 2 - github/workflows/minimal.yml | 31 ------- github/workflows/msrv.yml | 21 ----- github/workflows/os-check.yml | 26 ------ github/workflows/scheduled-nightly.yml | 1 - github/workflows/scheduled.yml | 61 ++++++++++++++ github/workflows/style.yml | 48 ----------- github/workflows/test.yml | 98 ++++++++++++++++++++-- 11 files changed, 261 insertions(+), 183 deletions(-) create mode 100644 github/workflows/check.yml delete mode 100644 github/workflows/coverage.yml delete mode 100644 github/workflows/features.yml delete mode 100644 github/workflows/locked.yml delete mode 100644 github/workflows/minimal.yml delete mode 100644 github/workflows/msrv.yml delete mode 100644 github/workflows/os-check.yml delete mode 100644 github/workflows/scheduled-nightly.yml create mode 100644 github/workflows/scheduled.yml delete mode 100644 github/workflows/style.yml diff --git a/github/workflows/check.yml b/github/workflows/check.yml new file mode 100644 index 0000000..81c3a01 --- /dev/null +++ b/github/workflows/check.yml @@ -0,0 +1,107 @@ +on: + push: + branches: [main] + pull_request: +name: check +jobs: + fmt: + runs-on: ubuntu-latest + name: stable / fmt + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install stable + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt + - name: cargo fmt --check + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --check + clippy: + runs-on: ubuntu-latest + name: ${{ matrix.toolchain }} / clippy + strategy: + fail-fast: false + matrix: + toolchain: [stable, beta] + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install ${{ matrix.toolchain }} + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.toolchain }} + default: true + components: clippy + - name: cargo clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + doc: + runs-on: ubuntu-latest + name: nightly / doc + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install nightly + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + default: true + - name: cargo doc + uses: actions-rs/cargo@v1 + with: + command: doc + args: --no-deps --all-features + env: + RUSTDOCFLAGS: --cfg docsrs + hack: + runs-on: ubuntu-latest + name: ubuntu / stable / features + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install stable + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: cargo install cargo-hack + uses: taiki-e/install-action@cargo-hack + - name: cargo hack + uses: actions-rs/cargo@v1 + with: + command: hack + args: --feature-powerset check --all-targets + msrv: + runs-on: ubuntu-latest + # we use a matrix here just because env can't be used in job names + # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability + strategy: + matrix: + msrv: [1.56.1] # 2021 edition requires 1.56 + name: ubuntu / ${{ matrix.msrv }} + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install ${{ matrix.toolchain }} + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.msrv }} + default: true + - name: cargo +${{ matrix.msrv }} check + uses: actions-rs/cargo@v1 + with: + command: check diff --git a/github/workflows/coverage.yml b/github/workflows/coverage.yml deleted file mode 100644 index 375f7a3..0000000 --- a/github/workflows/coverage.yml +++ /dev/null @@ -1,26 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: coverage -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - components: llvm-tools-preview - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - name: Generate code coverage - run: cargo llvm-cov --all-features --lcov --output-path lcov.info - - name: Upload to codecov.io - uses: codecov/codecov-action@v2 - with: - fail_ci_if_error: true diff --git a/github/workflows/features.yml b/github/workflows/features.yml deleted file mode 100644 index ac5e18e..0000000 --- a/github/workflows/features.yml +++ /dev/null @@ -1,23 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: cargo hack -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install cargo-hack - uses: taiki-e/install-action@cargo-hack - - name: cargo hack - uses: actions-rs/cargo@v1 - with: - command: hack - args: --feature-powerset check --all-targets diff --git a/github/workflows/locked.yml b/github/workflows/locked.yml deleted file mode 100644 index c67a317..0000000 --- a/github/workflows/locked.yml +++ /dev/null @@ -1,2 +0,0 @@ -# TODO: https://twitter.com/jonhoo/status/1571290371124260865 -# Maybe also: https://twitter.com/alcuadrado/status/1571291687837732873 diff --git a/github/workflows/minimal.yml b/github/workflows/minimal.yml deleted file mode 100644 index 0a558a9..0000000 --- a/github/workflows/minimal.yml +++ /dev/null @@ -1,31 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: With dependencies at minimal versions -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - uses: actions/checkout@v3 - with: - submodules: true - - name: cargo update -Zminimal-versions - uses: actions-rs/cargo@v1 - with: - command: update - toolchain: nightly - args: -Zminimal-versions - - name: cargo test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-features --all-targets diff --git a/github/workflows/msrv.yml b/github/workflows/msrv.yml deleted file mode 100644 index d6d9046..0000000 --- a/github/workflows/msrv.yml +++ /dev/null @@ -1,21 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: Minimum Supported Rust Version -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: 1.56.1 # 2021 edition requires 1.56 - override: true - - uses: actions/checkout@v3 - with: - submodules: true - - name: cargo +1.56.1 check - uses: actions-rs/cargo@v1 - with: - command: check diff --git a/github/workflows/os-check.yml b/github/workflows/os-check.yml deleted file mode 100644 index db3590b..0000000 --- a/github/workflows/os-check.yml +++ /dev/null @@ -1,26 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: os check -jobs: - os-check: - runs-on: ${{ matrix.os }} - name: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest, windows-latest] - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - uses: actions/checkout@v3 - with: - submodules: true - - name: cargo test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-features --all-targets diff --git a/github/workflows/scheduled-nightly.yml b/github/workflows/scheduled-nightly.yml deleted file mode 100644 index 5a86cca..0000000 --- a/github/workflows/scheduled-nightly.yml +++ /dev/null @@ -1 +0,0 @@ -# TODO: https://twitter.com/mycoliza/status/1571295690063753218 diff --git a/github/workflows/scheduled.yml b/github/workflows/scheduled.yml new file mode 100644 index 0000000..bb39a22 --- /dev/null +++ b/github/workflows/scheduled.yml @@ -0,0 +1,61 @@ +on: + push: + branches: [main] + pull_request: + schedule: + - cron: '7 7 * * *' +name: cargo test (rolling) +jobs: + # https://twitter.com/mycoliza/status/1571295690063753218 + nightly: + runs-on: ubuntu-latest + name: ubuntu / nightly + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install nightly + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + default: true + - name: cargo generate-lockfile + if: hashFiles('Cargo.lock') == '' + uses: actions-rs/cargo@v1 + with: + command: generate-lockfile + - name: cargo test --locked + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --all-features --all-targets + # https://twitter.com/alcuadrado/status/1571291687837732873 + update: + runs-on: ubuntu-latest + name: ubuntu / stable / updated + # There's no point running this if no Cargo.lock was checked in in the + # first place, since we'd just redo what happened in the regular test job. + # Unfortunately, hashFiles only works in if on steps, so we reepeat it. + # if: hashFiles('Cargo.lock') != '' + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install stable + if: hashFiles('Cargo.lock') != '' + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: cargo update + if: hashFiles('Cargo.lock') != '' + uses: actions-rs/cargo@v1 + with: + command: update + - name: cargo test + if: hashFiles('Cargo.lock') != '' + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --all-features --all-targets diff --git a/github/workflows/style.yml b/github/workflows/style.yml deleted file mode 100644 index 7a583f2..0000000 --- a/github/workflows/style.yml +++ /dev/null @@ -1,48 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: lint -jobs: - style: - runs-on: ubuntu-latest - name: ${{ matrix.toolchain }} - strategy: - fail-fast: false - matrix: - toolchain: [stable, beta] - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.toolchain }} - components: rustfmt, clippy - - uses: actions/checkout@v3 - with: - submodules: true - - name: cargo fmt --check - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --check - - name: cargo clippy - uses: actions-rs/clippy-check@v1 - if: always() - with: - token: ${{ secrets.GITHUB_TOKEN }} - doc: - runs-on: ubuntu-latest - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - - uses: actions/checkout@v3 - - name: cargo doc - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: doc - args: --no-deps --all-features - env: - RUSTDOCFLAGS: --cfg docsrs diff --git a/github/workflows/test.yml b/github/workflows/test.yml index dfc1819..83645f8 100644 --- a/github/workflows/test.yml +++ b/github/workflows/test.yml @@ -2,24 +2,112 @@ on: push: branches: [main] pull_request: -name: cargo test +name: test jobs: - test: + required: runs-on: ubuntu-latest name: ubuntu / ${{ matrix.toolchain }} strategy: matrix: - toolchain: [stable, beta, nightly] + toolchain: [stable, beta] steps: - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install ${{ matrix.toolchain }} + uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: ${{ matrix.toolchain }} + default: true + - name: cargo generate-lockfile + if: hashFiles('Cargo.lock') == '' + uses: actions-rs/cargo@v1 + with: + command: generate-lockfile + # https://twitter.com/jonhoo/status/1571290371124260865 + - name: cargo test --locked + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --all-features --all-targets + minimal: + runs-on: ubuntu-latest + name: ubuntu / stable / minimal-versions + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install stable + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: Install nightly for -Zminimal-versions + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + - name: cargo update -Zminimal-versions + uses: actions-rs/cargo@v1 + with: + command: update + toolchain: nightly + args: -Zminimal-versions + - name: cargo test + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked --all-features --all-targets + os-check: + runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} / stable + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + steps: - uses: actions/checkout@v3 with: submodules: true + - name: Install stable + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - name: cargo generate-lockfile + if: hashFiles('Cargo.lock') == '' + uses: actions-rs/cargo@v1 + with: + command: generate-lockfile - name: cargo test uses: actions-rs/cargo@v1 with: command: test - args: --all-features --all-targets + args: --locked --all-features --all-targets + coverage: + runs-on: ubuntu-latest + name: ubuntu / stable / coverage + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install stable + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: llvm-tools-preview + - name: cargo install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: cargo generate-lockfile + if: hashFiles('Cargo.lock') == '' + uses: actions-rs/cargo@v1 + with: + command: generate-lockfile + - name: cargo llvm-cov + run: cargo llvm-cov --locked --all-features --lcov --output-path lcov.info + - name: Upload to codecov.io + uses: codecov/codecov-action@v2 + with: + fail_ci_if_error: true From 71c2048cc0017a84a294be69d3b1629f55b1c8f0 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 18 Sep 2022 11:44:55 -0700 Subject: [PATCH 038/105] mv github .github This should make it possible to have rust-ci-conf as a remote you merge from. --- {github => .github}/codecov.yml | 0 {github => .github}/workflows/asan.yml | 0 {github => .github}/workflows/check.yml | 0 {github => .github}/workflows/loom.yml | 0 {github => .github}/workflows/lsan.yml | 0 {github => .github}/workflows/miri.yml | 0 {github => .github}/workflows/nostd.yml | 0 {github => .github}/workflows/scheduled.yml | 0 {github => .github}/workflows/test.yml | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename {github => .github}/codecov.yml (100%) rename {github => .github}/workflows/asan.yml (100%) rename {github => .github}/workflows/check.yml (100%) rename {github => .github}/workflows/loom.yml (100%) rename {github => .github}/workflows/lsan.yml (100%) rename {github => .github}/workflows/miri.yml (100%) rename {github => .github}/workflows/nostd.yml (100%) rename {github => .github}/workflows/scheduled.yml (100%) rename {github => .github}/workflows/test.yml (100%) diff --git a/github/codecov.yml b/.github/codecov.yml similarity index 100% rename from github/codecov.yml rename to .github/codecov.yml diff --git a/github/workflows/asan.yml b/.github/workflows/asan.yml similarity index 100% rename from github/workflows/asan.yml rename to .github/workflows/asan.yml diff --git a/github/workflows/check.yml b/.github/workflows/check.yml similarity index 100% rename from github/workflows/check.yml rename to .github/workflows/check.yml diff --git a/github/workflows/loom.yml b/.github/workflows/loom.yml similarity index 100% rename from github/workflows/loom.yml rename to .github/workflows/loom.yml diff --git a/github/workflows/lsan.yml b/.github/workflows/lsan.yml similarity index 100% rename from github/workflows/lsan.yml rename to .github/workflows/lsan.yml diff --git a/github/workflows/miri.yml b/.github/workflows/miri.yml similarity index 100% rename from github/workflows/miri.yml rename to .github/workflows/miri.yml diff --git a/github/workflows/nostd.yml b/.github/workflows/nostd.yml similarity index 100% rename from github/workflows/nostd.yml rename to .github/workflows/nostd.yml diff --git a/github/workflows/scheduled.yml b/.github/workflows/scheduled.yml similarity index 100% rename from github/workflows/scheduled.yml rename to .github/workflows/scheduled.yml diff --git a/github/workflows/test.yml b/.github/workflows/test.yml similarity index 100% rename from github/workflows/test.yml rename to .github/workflows/test.yml From 56d4398a24f8c7aae0ba4a74eefaf75d1c3db3a8 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 18 Sep 2022 11:52:12 -0700 Subject: [PATCH 039/105] Merge safety workflows --- .github/workflows/asan.yml | 24 ---------- .github/workflows/loom.yml | 22 --------- .github/workflows/lsan.yml | 32 -------------- .github/workflows/miri.yml | 25 ----------- .github/workflows/safety.yml | 86 ++++++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 103 deletions(-) delete mode 100644 .github/workflows/asan.yml delete mode 100644 .github/workflows/loom.yml delete mode 100644 .github/workflows/lsan.yml delete mode 100644 .github/workflows/miri.yml create mode 100644 .github/workflows/safety.yml diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml deleted file mode 100644 index 5604b7e..0000000 --- a/.github/workflows/asan.yml +++ /dev/null @@ -1,24 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: Address sanitizer -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - uses: actions/checkout@v3 - - name: cargo test -Zsanitizer=address - uses: actions-rs/cargo@v1 - with: - command: test - # only --lib --tests b/c of https://github.com/rust-lang/rust/issues/53945 - args: --lib --tests --all-features --target x86_64-unknown-linux-gnu - env: - ASAN_OPTIONS: "detect_odr_violation=0:detect_leaks=0" - RUSTFLAGS: "-Z sanitizer=address" diff --git a/.github/workflows/loom.yml b/.github/workflows/loom.yml deleted file mode 100644 index 8cfe45d..0000000 --- a/.github/workflows/loom.yml +++ /dev/null @@ -1,22 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: Loom -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - - uses: actions/checkout@v3 - - name: cargo test --test loom - uses: actions-rs/cargo@v1 - with: - command: test - args: --release --test loom - env: - LOOM_MAX_PREEMPTIONS: 2 - RUSTFLAGS: "--cfg loom" diff --git a/.github/workflows/lsan.yml b/.github/workflows/lsan.yml deleted file mode 100644 index a966913..0000000 --- a/.github/workflows/lsan.yml +++ /dev/null @@ -1,32 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: Leak sanitizer -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - uses: actions/checkout@v3 - - run: | - # to get the symbolizer for debug symbol resolution - sudo apt install llvm - # to fix buggy leak analyzer: - # https://github.com/japaric/rust-san#unrealiable-leaksanitizer - sed -i '/\[features\]/i [profile.dev]' Cargo.toml - sed -i '/profile.dev/a opt-level = 1' Cargo.toml - cat Cargo.toml - name: Enable debug symbols - - name: cargo test -Zsanitizer=leak - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-features --target x86_64-unknown-linux-gnu - env: - RUSTFLAGS: "-Z sanitizer=leak" - LSAN_OPTIONS: "suppressions=lsan-suppressions.txt" diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml deleted file mode 100644 index ae9539a..0000000 --- a/.github/workflows/miri.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: - push: - branches: [main] - pull_request: -name: Miri -jobs: - test: - runs-on: ubuntu-latest - steps: - - run: | - echo "NIGHTLY=nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/miri)" >> $GITHUB_ENV - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ env.NIGHTLY }} - override: true - components: miri - - uses: actions/checkout@v3 - - name: cargo miri test - uses: actions-rs/cargo@v1 - with: - command: miri - args: test - env: - MIRIFLAGS: "-Zmiri-tag-raw-pointers" diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml new file mode 100644 index 0000000..86fc9ee --- /dev/null +++ b/.github/workflows/safety.yml @@ -0,0 +1,86 @@ +on: + push: + branches: [main] + pull_request: +name: safety +jobs: + sanitizers: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install nightly + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + default: true + - run: | + # to get the symbolizer for debug symbol resolution + sudo apt install llvm + # to fix buggy leak analyzer: + # https://github.com/japaric/rust-san#unrealiable-leaksanitizer + sed -i '/\[features\]/i [profile.dev]' Cargo.toml + sed -i '/profile.dev/a opt-level = 1' Cargo.toml + cat Cargo.toml + name: Enable debug symbols + - name: cargo test -Zsanitizer=address + uses: actions-rs/cargo@v1 + with: + command: test + # only --lib --tests b/c of https://github.com/rust-lang/rust/issues/53945 + args: --lib --tests --all-features --target x86_64-unknown-linux-gnu + env: + ASAN_OPTIONS: "detect_odr_violation=0:detect_leaks=0" + RUSTFLAGS: "-Z sanitizer=address" + - name: cargo test -Zsanitizer=leak + if: always() + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-features --target x86_64-unknown-linux-gnu + env: + LSAN_OPTIONS: "suppressions=lsan-suppressions.txt" + RUSTFLAGS: "-Z sanitizer=leak" + miri: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - run: | + echo "NIGHTLY=nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/miri)" >> $GITHUB_ENV + - name: Install ${{ env.NIGHTLY }} + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ env.NIGHTLY }} + default: true + components: miri + - name: cargo miri test + uses: actions-rs/cargo@v1 + with: + command: miri + args: test + env: + MIRIFLAGS: "-Zmiri-tag-raw-pointers" + loom: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install stable + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + - name: cargo test --test loom + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --test loom + env: + LOOM_MAX_PREEMPTIONS: 2 + RUSTFLAGS: "--cfg loom" From 15c1fa2ffcc0f31fabcdcd90cde6a05b54baf8b5 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 18 Sep 2022 12:10:58 -0700 Subject: [PATCH 040/105] Catch upcoming deprecations --- .github/workflows/scheduled.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index bb39a22..20229ba 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -33,7 +33,7 @@ jobs: # https://twitter.com/alcuadrado/status/1571291687837732873 update: runs-on: ubuntu-latest - name: ubuntu / stable / updated + name: ubuntu / beta / updated # There's no point running this if no Cargo.lock was checked in in the # first place, since we'd just redo what happened in the regular test job. # Unfortunately, hashFiles only works in if on steps, so we reepeat it. @@ -42,12 +42,13 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Install stable + - name: Install beta if: hashFiles('Cargo.lock') != '' uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: stable + toolchain: beta + default: true - name: cargo update if: hashFiles('Cargo.lock') != '' uses: actions-rs/cargo@v1 @@ -59,3 +60,5 @@ jobs: with: command: test args: --locked --all-features --all-targets + env: + RUSTFLAGS: -D deprecated From ea198cc4991e2f869cd99cb8175652576ef15119 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 18 Sep 2022 13:52:47 -0700 Subject: [PATCH 041/105] More concise name for scheduled jobs --- .github/workflows/scheduled.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 20229ba..a5f5311 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -4,7 +4,7 @@ on: pull_request: schedule: - cron: '7 7 * * *' -name: cargo test (rolling) +name: rolling jobs: # https://twitter.com/mycoliza/status/1571295690063753218 nightly: From 441dc27e4d1e365bfc9b0c25e736da6cb1d15102 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 18 Sep 2022 14:01:04 -0700 Subject: [PATCH 042/105] Allow examples and binaries to require features --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 81c3a01..560e9e2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -82,7 +82,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: hack - args: --feature-powerset check --all-targets + args: --feature-powerset check --lib --tests msrv: runs-on: ubuntu-latest # we use a matrix here just because env can't be used in job names From b783cb31ab3c6c27ad826bde44aa917c0d0908da Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 23 Sep 2022 08:53:07 -0700 Subject: [PATCH 043/105] Use dependabot, but only for major versions --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..22cfb43 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: daily + ignore: + dependency-name: "*" + # patch and minor updates don't matter for libraries + # remove this ignore rule if your package has binaries + update-types: + - "version-update:semver-patch" + - "version-update:semver-minor" From cf47d4cad4b241a30245a51aa1ac7e99e7fedf8a Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Wed, 28 Sep 2022 18:23:39 -0700 Subject: [PATCH 044/105] ignore is a list --- .github/dependabot.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 22cfb43..9c60153 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,9 @@ updates: schedule: interval: daily ignore: - dependency-name: "*" - # patch and minor updates don't matter for libraries - # remove this ignore rule if your package has binaries - update-types: - - "version-update:semver-patch" - - "version-update:semver-minor" + - dependency-name: "*" + # patch and minor updates don't matter for libraries + # remove this ignore rule if your package has binaries + update-types: + - "version-update:semver-patch" + - "version-update:semver-minor" From 82cbed84f82e8538cdfc99dcf1b8b2cbab4fb126 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 9 Dec 2022 16:25:21 -0800 Subject: [PATCH 045/105] Notify if actions themselves are outdated --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9c60153..8139a93 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,9 @@ version: 2 updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily - package-ecosystem: cargo directory: / schedule: From c8a7835b2f0b21d9a64e6a8b0ddc10fbc88e2dd1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Dec 2022 00:25:41 +0000 Subject: [PATCH 046/105] Bump codecov/codecov-action from 2 to 3 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 2 to 3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v2...v3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 83645f8..9569167 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,6 +108,6 @@ jobs: - name: cargo llvm-cov run: cargo llvm-cov --locked --all-features --lcov --output-path lcov.info - name: Upload to codecov.io - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: fail_ci_if_error: true From 362696ab8007ef1a4779885a398286856cacf555 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 10 Mar 2023 21:16:35 -0800 Subject: [PATCH 047/105] Move to maintained rust installer See https://github.com/actions-rs/toolchain/issues/216 --- .github/workflows/check.yml | 23 +++++------------------ .github/workflows/nostd.yml | 11 ++++++----- .github/workflows/safety.yml | 15 +++------------ .github/workflows/scheduled.yml | 12 ++---------- .github/workflows/test.yml | 25 +++++++------------------ 5 files changed, 23 insertions(+), 63 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 560e9e2..da175c2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -12,10 +12,8 @@ jobs: with: submodules: true - name: Install stable - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: - profile: minimal - toolchain: stable components: rustfmt - name: cargo fmt --check uses: actions-rs/cargo@v1 @@ -34,11 +32,9 @@ jobs: with: submodules: true - name: Install ${{ matrix.toolchain }} - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: ${{ matrix.toolchain }} - default: true components: clippy - name: cargo clippy uses: actions-rs/clippy-check@v1 @@ -52,11 +48,7 @@ jobs: with: submodules: true - name: Install nightly - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - default: true + uses: dtolnay/rust-toolchain@nightly - name: cargo doc uses: actions-rs/cargo@v1 with: @@ -72,10 +64,7 @@ jobs: with: submodules: true - name: Install stable - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable + uses: dtolnay/rust-toolchain@stable - name: cargo install cargo-hack uses: taiki-e/install-action@cargo-hack - name: cargo hack @@ -96,11 +85,9 @@ jobs: with: submodules: true - name: Install ${{ matrix.toolchain }} - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: ${{ matrix.msrv }} - default: true - name: cargo +${{ matrix.msrv }} check uses: actions-rs/cargo@v1 with: diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml index 409ae73..235df26 100644 --- a/.github/workflows/nostd.yml +++ b/.github/workflows/nostd.yml @@ -11,12 +11,13 @@ jobs: matrix: target: [thumbv7m-none-eabi, aarch64-unknown-none] steps: - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - target: ${{ matrix.target }} - uses: actions/checkout@v3 + with: + submodules: true + - name: Install stable + uses: dtolnay/rust-toolchain@stable + - name: rustup target add ${{ matrix.target }} + run: rustup target add ${{ matrix.target }} - name: cargo check uses: actions-rs/cargo@v1 with: diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index 86fc9ee..77f56a4 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -11,11 +11,7 @@ jobs: with: submodules: true - name: Install nightly - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - default: true + uses: dtolnay/rust-toolchain@nightly - run: | # to get the symbolizer for debug symbol resolution sudo apt install llvm @@ -52,11 +48,9 @@ jobs: - run: | echo "NIGHTLY=nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/miri)" >> $GITHUB_ENV - name: Install ${{ env.NIGHTLY }} - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: ${{ env.NIGHTLY }} - default: true components: miri - name: cargo miri test uses: actions-rs/cargo@v1 @@ -72,10 +66,7 @@ jobs: with: submodules: true - name: Install stable - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal + uses: dtolnay/rust-toolchain@stable - name: cargo test --test loom uses: actions-rs/cargo@v1 with: diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index a5f5311..e341e9b 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -15,11 +15,7 @@ jobs: with: submodules: true - name: Install nightly - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - default: true + uses: dtolnay/rust-toolchain@nightly - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' uses: actions-rs/cargo@v1 @@ -44,11 +40,7 @@ jobs: submodules: true - name: Install beta if: hashFiles('Cargo.lock') != '' - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: beta - default: true + uses: dtolnay/rust-toolchain@beta - name: cargo update if: hashFiles('Cargo.lock') != '' uses: actions-rs/cargo@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9569167..53db840 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,11 +15,9 @@ jobs: with: submodules: true - name: Install ${{ matrix.toolchain }} - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: ${{ matrix.toolchain }} - default: true - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' uses: actions-rs/cargo@v1 @@ -39,15 +37,11 @@ jobs: with: submodules: true - name: Install stable - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable + uses: dtolnay/rust-toolchain@stable - name: Install nightly for -Zminimal-versions - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly + uses: dtolnay/rust-toolchain@nightly + - name: rustup default stable + run: rustup default stable - name: cargo update -Zminimal-versions uses: actions-rs/cargo@v1 with: @@ -71,10 +65,7 @@ jobs: with: submodules: true - name: Install stable - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable + uses: dtolnay/rust-toolchain@stable - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' uses: actions-rs/cargo@v1 @@ -93,10 +84,8 @@ jobs: with: submodules: true - name: Install stable - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: - profile: minimal - toolchain: stable components: llvm-tools-preview - name: cargo install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov From 90999e1bd1a9dabaecd149697f69e8e26e810562 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Fri, 10 Mar 2023 21:22:30 -0800 Subject: [PATCH 048/105] Fix install message for msrv --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index da175c2..d25da88 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -84,7 +84,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Install ${{ matrix.toolchain }} + - name: Install ${{ matrix.msrv }} uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.msrv }} From 9afb0e111adcd678ef06884cf737aa9e0cf135e7 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 11 Mar 2023 15:07:39 -0800 Subject: [PATCH 049/105] Get rid of most actions-rs bits Given that that project is unmaintained. https://github.com/actions-rs/toolchain/issues/216 --- .github/workflows/check.yml | 19 ++++--------------- .github/workflows/nostd.yml | 5 +---- .github/workflows/safety.yml | 22 +++++----------------- .github/workflows/scheduled.yml | 18 ++++-------------- .github/workflows/test.yml | 33 +++++++-------------------------- 5 files changed, 21 insertions(+), 76 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d25da88..29fc888 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -16,10 +16,7 @@ jobs: with: components: rustfmt - name: cargo fmt --check - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --check + run: cargo fmt --check clippy: runs-on: ubuntu-latest name: ${{ matrix.toolchain }} / clippy @@ -50,10 +47,7 @@ jobs: - name: Install nightly uses: dtolnay/rust-toolchain@nightly - name: cargo doc - uses: actions-rs/cargo@v1 - with: - command: doc - args: --no-deps --all-features + run: cargo doc --no-deps --all-features env: RUSTDOCFLAGS: --cfg docsrs hack: @@ -68,10 +62,7 @@ jobs: - name: cargo install cargo-hack uses: taiki-e/install-action@cargo-hack - name: cargo hack - uses: actions-rs/cargo@v1 - with: - command: hack - args: --feature-powerset check --lib --tests + run: cargo hack --feature-powerset check --lib --tests msrv: runs-on: ubuntu-latest # we use a matrix here just because env can't be used in job names @@ -89,6 +80,4 @@ jobs: with: toolchain: ${{ matrix.msrv }} - name: cargo +${{ matrix.msrv }} check - uses: actions-rs/cargo@v1 - with: - command: check + run: cargo check diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml index 235df26..397a2da 100644 --- a/.github/workflows/nostd.yml +++ b/.github/workflows/nostd.yml @@ -19,7 +19,4 @@ jobs: - name: rustup target add ${{ matrix.target }} run: rustup target add ${{ matrix.target }} - name: cargo check - uses: actions-rs/cargo@v1 - with: - command: check - args: --target ${{ matrix.target }} --no-default-features + run: cargo check --target ${{ matrix.target }} --no-default-features diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index 77f56a4..6c973c0 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -22,20 +22,14 @@ jobs: cat Cargo.toml name: Enable debug symbols - name: cargo test -Zsanitizer=address - uses: actions-rs/cargo@v1 - with: - command: test - # only --lib --tests b/c of https://github.com/rust-lang/rust/issues/53945 - args: --lib --tests --all-features --target x86_64-unknown-linux-gnu + # only --lib --tests b/c of https://github.com/rust-lang/rust/issues/53945 + run: cargo test --lib --tests --all-features --target x86_64-unknown-linux-gnu env: ASAN_OPTIONS: "detect_odr_violation=0:detect_leaks=0" RUSTFLAGS: "-Z sanitizer=address" - name: cargo test -Zsanitizer=leak if: always() - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-features --target x86_64-unknown-linux-gnu + run: cargo test --all-features --target x86_64-unknown-linux-gnu env: LSAN_OPTIONS: "suppressions=lsan-suppressions.txt" RUSTFLAGS: "-Z sanitizer=leak" @@ -53,10 +47,7 @@ jobs: toolchain: ${{ env.NIGHTLY }} components: miri - name: cargo miri test - uses: actions-rs/cargo@v1 - with: - command: miri - args: test + run: cargo miri test env: MIRIFLAGS: "-Zmiri-tag-raw-pointers" loom: @@ -68,10 +59,7 @@ jobs: - name: Install stable uses: dtolnay/rust-toolchain@stable - name: cargo test --test loom - uses: actions-rs/cargo@v1 - with: - command: test - args: --release --test loom + run: cargo test --release --test loom env: LOOM_MAX_PREEMPTIONS: 2 RUSTFLAGS: "--cfg loom" diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index e341e9b..2c15695 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -18,14 +18,9 @@ jobs: uses: dtolnay/rust-toolchain@nightly - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' - uses: actions-rs/cargo@v1 - with: - command: generate-lockfile + run: cargo generate-lockfile - name: cargo test --locked - uses: actions-rs/cargo@v1 - with: - command: test - args: --locked --all-features --all-targets + run: cargo test --locked --all-features --all-targets # https://twitter.com/alcuadrado/status/1571291687837732873 update: runs-on: ubuntu-latest @@ -43,14 +38,9 @@ jobs: uses: dtolnay/rust-toolchain@beta - name: cargo update if: hashFiles('Cargo.lock') != '' - uses: actions-rs/cargo@v1 - with: - command: update + run: cargo update - name: cargo test if: hashFiles('Cargo.lock') != '' - uses: actions-rs/cargo@v1 - with: - command: test - args: --locked --all-features --all-targets + run: cargo test --locked --all-features --all-targets env: RUSTFLAGS: -D deprecated diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53db840..4435fd7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,15 +20,10 @@ jobs: toolchain: ${{ matrix.toolchain }} - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' - uses: actions-rs/cargo@v1 - with: - command: generate-lockfile + run: cargo generate-lockfile # https://twitter.com/jonhoo/status/1571290371124260865 - name: cargo test --locked - uses: actions-rs/cargo@v1 - with: - command: test - args: --locked --all-features --all-targets + run: cargo test --locked --all-features --all-targets minimal: runs-on: ubuntu-latest name: ubuntu / stable / minimal-versions @@ -43,16 +38,9 @@ jobs: - name: rustup default stable run: rustup default stable - name: cargo update -Zminimal-versions - uses: actions-rs/cargo@v1 - with: - command: update - toolchain: nightly - args: -Zminimal-versions + run: cargo +nightly update -Zminimal-versions - name: cargo test - uses: actions-rs/cargo@v1 - with: - command: test - args: --locked --all-features --all-targets + run: cargo test --locked --all-features --all-targets os-check: runs-on: ${{ matrix.os }} name: ${{ matrix.os }} / stable @@ -68,14 +56,9 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' - uses: actions-rs/cargo@v1 - with: - command: generate-lockfile + run: cargo generate-lockfile - name: cargo test - uses: actions-rs/cargo@v1 - with: - command: test - args: --locked --all-features --all-targets + run: cargo test --locked --all-features --all-targets coverage: runs-on: ubuntu-latest name: ubuntu / stable / coverage @@ -91,9 +74,7 @@ jobs: uses: taiki-e/install-action@cargo-llvm-cov - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' - uses: actions-rs/cargo@v1 - with: - command: generate-lockfile + run: cargo generate-lockfile - name: cargo llvm-cov run: cargo llvm-cov --locked --all-features --lcov --output-path lcov.info - name: Upload to codecov.io From a076ec1cb42e88e6444ae7f573570ec53c149074 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 11 Mar 2023 15:08:45 -0800 Subject: [PATCH 050/105] Minimal token permissions See https://github.com/tokio-rs/tokio/pull/5072 --- .github/workflows/check.yml | 5 +++++ .github/workflows/nostd.yml | 2 ++ .github/workflows/safety.yml | 2 ++ .github/workflows/scheduled.yml | 2 ++ .github/workflows/test.yml | 2 ++ 5 files changed, 13 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 29fc888..406ba9b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,3 +1,5 @@ +permissions: + contents: read on: push: branches: [main] @@ -20,6 +22,9 @@ jobs: clippy: runs-on: ubuntu-latest name: ${{ matrix.toolchain }} / clippy + permissions: + contents: read + checks: write strategy: fail-fast: false matrix: diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml index 397a2da..7efae0f 100644 --- a/.github/workflows/nostd.yml +++ b/.github/workflows/nostd.yml @@ -1,3 +1,5 @@ +permissions: + contents: read on: push: branches: [main] diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index 6c973c0..4fa7ad7 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -1,3 +1,5 @@ +permissions: + contents: read on: push: branches: [main] diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 2c15695..0215432 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1,3 +1,5 @@ +permissions: + contents: read on: push: branches: [main] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4435fd7..8aa4488 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,3 +1,5 @@ +permissions: + contents: read on: push: branches: [main] From 5ea59356dc9379a08dff5bf3df3c5016df2ca7f3 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 12 Mar 2023 09:40:09 -0700 Subject: [PATCH 051/105] Remove -Zmiri-tag-raw-pointers as it's now default --- .github/workflows/safety.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index 4fa7ad7..edf40e5 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -51,7 +51,7 @@ jobs: - name: cargo miri test run: cargo miri test env: - MIRIFLAGS: "-Zmiri-tag-raw-pointers" + MIRIFLAGS: "" loom: runs-on: ubuntu-latest steps: From 0d12c82bf4a89014643cd2b7991c63da9dd8b15b Mon Sep 17 00:00:00 2001 From: Tudyx <56633664+Tudyx@users.noreply.github.com> Date: Mon, 20 Mar 2023 01:17:59 +0100 Subject: [PATCH 052/105] Unbreak cargo hack for non-libraries (#4) --- .github/workflows/check.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 406ba9b..a687dfa 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -66,8 +66,9 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: cargo install cargo-hack uses: taiki-e/install-action@cargo-hack + # intentionally no target specifier; see https://github.com/jonhoo/rust-ci-conf/pull/4 - name: cargo hack - run: cargo hack --feature-powerset check --lib --tests + run: cargo hack --feature-powerset check msrv: runs-on: ubuntu-latest # we use a matrix here just because env can't be used in job names From 80a89195f2ac9971ecff9d422a6bb83b3f84e0bc Mon Sep 17 00:00:00 2001 From: Burkhard Mittelbach Date: Mon, 20 Mar 2023 01:18:49 +0100 Subject: [PATCH 053/105] Add action to run doctest. (#3) `cargo test --all-features` does not run doc-tests. For more information see https://github.com/rust-lang/cargo/issues/6669. --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8aa4488..c675b38 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,9 @@ jobs: # https://twitter.com/jonhoo/status/1571290371124260865 - name: cargo test --locked run: cargo test --locked --all-features --all-targets + # https://github.com/rust-lang/cargo/issues/6669 + - name: cargo test --doc + run: cargo test --locked --all-features --doc minimal: runs-on: ubuntu-latest name: ubuntu / stable / minimal-versions From 16a2c2925eb46e24208b20bca567f1e7546f4e2f Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Sat, 8 Apr 2023 18:57:54 +0200 Subject: [PATCH 054/105] chore: automatically cancel superseded Actions runs (#5) --- .github/workflows/check.yml | 4 ++++ .github/workflows/nostd.yml | 4 ++++ .github/workflows/safety.yml | 4 ++++ .github/workflows/scheduled.yml | 4 ++++ .github/workflows/test.yml | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a687dfa..3fb97ad 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -4,6 +4,10 @@ on: push: branches: [main] pull_request: +# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true name: check jobs: fmt: diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml index 7efae0f..32d56c3 100644 --- a/.github/workflows/nostd.yml +++ b/.github/workflows/nostd.yml @@ -4,6 +4,10 @@ on: push: branches: [main] pull_request: +# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true name: no-std jobs: nostd: diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index edf40e5..9c7aeee 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -4,6 +4,10 @@ on: push: branches: [main] pull_request: +# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true name: safety jobs: sanitizers: diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 0215432..d4e19dd 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -6,6 +6,10 @@ on: pull_request: schedule: - cron: '7 7 * * *' +# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true name: rolling jobs: # https://twitter.com/mycoliza/status/1571295690063753218 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c675b38..4d0417c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,10 @@ on: push: branches: [main] pull_request: +# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true name: test jobs: required: From 6332a3af21a58f811a681a98cd44d0f5da8a1891 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Mon, 24 Apr 2023 12:40:14 -0700 Subject: [PATCH 055/105] [sanity] More robust injection of opt-level 1 (#9) Fixes #8 --- .github/workflows/safety.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index 9c7aeee..eb5a5d6 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -23,8 +23,15 @@ jobs: sudo apt install llvm # to fix buggy leak analyzer: # https://github.com/japaric/rust-san#unrealiable-leaksanitizer - sed -i '/\[features\]/i [profile.dev]' Cargo.toml - sed -i '/profile.dev/a opt-level = 1' Cargo.toml + # ensure there's a profile.dev section + if ! grep -qE '^[ \t]*[profile.dev]' Cargo.toml; then + echo >> Cargo.toml + echo '[profile.dev]' >> Cargo.toml + fi + # remove pre-existing opt-levels in profile.dev + sed -i '/^\s*\[profile.dev\]/,/^\s*\[/ {/^\s*opt-level/d}' Cargo.toml + # now set opt-level to 1 + sed -i '/^\s*\[profile.dev\]/a opt-level = 1' Cargo.toml cat Cargo.toml name: Enable debug symbols - name: cargo test -Zsanitizer=address From 99f108f93c4f09906a4ddd4506fa4a2cbc68169d Mon Sep 17 00:00:00 2001 From: James Chacon Date: Sun, 13 Aug 2023 03:13:38 -0700 Subject: [PATCH 056/105] Quote MSRV version to avoid float parsing (#11) Put 1.70 in there (for instance if you want to pin against OnceLock stabilizing) and it will actually test 1.7 as it appears github auto converts this to a float? Putting in quotes seems to do the right thing here --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3fb97ad..6b136dc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -79,7 +79,7 @@ jobs: # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability strategy: matrix: - msrv: [1.56.1] # 2021 edition requires 1.56 + msrv: ["1.56.1"] # 2021 edition requires 1.56 name: ubuntu / ${{ matrix.msrv }} steps: - uses: actions/checkout@v3 From c704bcc656871d37f93f79234c4d530522ac8733 Mon Sep 17 00:00:00 2001 From: Rod Elias Date: Sun, 20 Aug 2023 09:40:15 -0300 Subject: [PATCH 057/105] Install Openssl for Windows (#12) --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d0417c..9389fd3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,6 +58,10 @@ jobs: matrix: os: [macos-latest, windows-latest] steps: + - run: echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append + if: runner.os == 'Windows' + - run: vcpkg install openssl:x64-windows-static-md + if: runner.os == 'Windows' - uses: actions/checkout@v3 with: submodules: true From 7c327ddf5b42f5309f564da8275c789c4a22fdbb Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 20 Aug 2023 15:12:25 +0200 Subject: [PATCH 058/105] Don't install OpenSSL on Windows by default --- .github/workflows/test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9389fd3..3ef2713 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,10 +58,12 @@ jobs: matrix: os: [macos-latest, windows-latest] steps: - - run: echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append - if: runner.os == 'Windows' - - run: vcpkg install openssl:x64-windows-static-md - if: runner.os == 'Windows' + # if your project needs OpenSSL, uncommment this to fix Windows builds. + # it's commented out by default as tthe install command takes 5-10m. + # - run: echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append + # if: runner.os == 'Windows' + # - run: vcpkg install openssl:x64-windows-static-md + # if: runner.os == 'Windows' - uses: actions/checkout@v3 with: submodules: true From deb9fd3f46de5eebd2583292362a161b1d1c56c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 16:21:14 +0200 Subject: [PATCH 059/105] Bump actions/checkout from 3 to 4 (#13) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/check.yml | 10 +++++----- .github/workflows/nostd.yml | 2 +- .github/workflows/safety.yml | 6 +++--- .github/workflows/scheduled.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6b136dc..3fdc139 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest name: stable / fmt steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install stable @@ -34,7 +34,7 @@ jobs: matrix: toolchain: [stable, beta] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install ${{ matrix.toolchain }} @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest name: nightly / doc steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install nightly @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-latest name: ubuntu / stable / features steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install stable @@ -82,7 +82,7 @@ jobs: msrv: ["1.56.1"] # 2021 edition requires 1.56 name: ubuntu / ${{ matrix.msrv }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install ${{ matrix.msrv }} diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml index 32d56c3..00362a6 100644 --- a/.github/workflows/nostd.yml +++ b/.github/workflows/nostd.yml @@ -17,7 +17,7 @@ jobs: matrix: target: [thumbv7m-none-eabi, aarch64-unknown-none] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install stable diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index eb5a5d6..ce1e38e 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -13,7 +13,7 @@ jobs: sanitizers: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install nightly @@ -49,7 +49,7 @@ jobs: miri: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - run: | @@ -66,7 +66,7 @@ jobs: loom: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install stable diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index d4e19dd..4607f3e 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest name: ubuntu / nightly steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install nightly @@ -36,7 +36,7 @@ jobs: # Unfortunately, hashFiles only works in if on steps, so we reepeat it. # if: hashFiles('Cargo.lock') != '' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install beta diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ef2713..56bd560 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: matrix: toolchain: [stable, beta] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install ${{ matrix.toolchain }} @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest name: ubuntu / stable / minimal-versions steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install stable @@ -64,7 +64,7 @@ jobs: # if: runner.os == 'Windows' # - run: vcpkg install openssl:x64-windows-static-md # if: runner.os == 'Windows' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install stable @@ -78,7 +78,7 @@ jobs: runs-on: ubuntu-latest name: ubuntu / stable / coverage steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install stable From bfee1175f6378c9191eca0af88e068ed49d48bef Mon Sep 17 00:00:00 2001 From: Josh McKinney Date: Sat, 11 Nov 2023 01:14:12 -0800 Subject: [PATCH 060/105] docs: Add documentation based on the youtube video (#10) --- .github/DOCS.md | 23 ++++++++++++++++++++ .github/codecov.yml | 2 +- .github/dependabot.yml | 6 ++++-- .github/workflows/check.yml | 21 ++++++++++++++++++- .github/workflows/nostd.yml | 4 +++- .github/workflows/safety.yml | 8 ++++++- .github/workflows/scheduled.yml | 16 +++++++++----- .github/workflows/test.yml | 37 ++++++++++++++++++++++++++++++++- 8 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 .github/DOCS.md diff --git a/.github/DOCS.md b/.github/DOCS.md new file mode 100644 index 0000000..e932784 --- /dev/null +++ b/.github/DOCS.md @@ -0,0 +1,23 @@ +# Github config and workflows + +In this folder there is configuration for codecoverage, dependabot, and ci +workflows that check the library more deeply than the default configurations. + +This folder can be or was merged using a --allow-unrelated-histories merge +strategy from which provides a +reasonably sensible base for writing your own ci on. By using this strategy +the history of the CI repo is included in your repo, and future updates to +the CI can be merged later. + +To perform this merge run: + +```shell +git remote add ci https://github.com/jonhoo/rust-ci-conf.git +git fetch ci +git merge --allow-unrelated-histories ci/main +``` + +An overview of the files in this project is available at: +, which contains some +rationale for decisions and runs through an example of solving minimal version +and OpenSSL issues. diff --git a/.github/codecov.yml b/.github/codecov.yml index ff4f571..cd5ce8f 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -18,4 +18,4 @@ ignore: # Make comments less noisy comment: layout: "files" - require_changes: yes + require_changes: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8139a93..d0f091e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,8 +10,10 @@ updates: interval: daily ignore: - dependency-name: "*" - # patch and minor updates don't matter for libraries - # remove this ignore rule if your package has binaries + # patch and minor updates don't matter for libraries as consumers of this library build + # with their own lockfile, rather than the version specified in this library's lockfile + # remove this ignore rule if your package has binaries to ensure that the binaries are + # built with the exact set of dependencies and those are up to date. update-types: - "version-update:semver-patch" - "version-update:semver-minor" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3fdc139..6b4d13a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,10 +1,21 @@ +# This workflow runs whenever a PR is opened or updated, or a commit is pushed to main. It runs +# several checks: +# - fmt: checks that the code is formatted according to rustfmt +# - clippy: checks that the code does not contain any clippy warnings +# - doc: checks that the code can be documented without errors +# - hack: check combinations of feature flags +# - msrv: check that the msrv specified in the crate is correct permissions: contents: read +# This configuration allows maintainers of this repo to create a branch and pull request based on +# the new branch. Restricting the push trigger to the main branch ensures that the PR only gets +# built once. on: push: branches: [main] pull_request: -# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 +# If new code is pushed to a PR branch, then cancel in progress workflows for that PR. Ensures that +# we don't waste CI time, and returns results quicker https://github.com/jonhoo/rust-ci-conf/pull/5 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -32,6 +43,7 @@ jobs: strategy: fail-fast: false matrix: + # Get early warning of new lints which are regularly introduced in beta channels. toolchain: [stable, beta] steps: - uses: actions/checkout@v4 @@ -47,6 +59,9 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} doc: + # run docs generation on nightly rather than stable. This enables features like + # https://doc.rust-lang.org/beta/unstable-book/language-features/doc-cfg.html which allows an + # API be documented as only available in some specific platforms. runs-on: ubuntu-latest name: nightly / doc steps: @@ -60,6 +75,8 @@ jobs: env: RUSTDOCFLAGS: --cfg docsrs hack: + # cargo-hack checks combinations of feature flags to ensure that features are all additive + # which is required for feature unification runs-on: ubuntu-latest name: ubuntu / stable / features steps: @@ -71,9 +88,11 @@ jobs: - name: cargo install cargo-hack uses: taiki-e/install-action@cargo-hack # intentionally no target specifier; see https://github.com/jonhoo/rust-ci-conf/pull/4 + # --feature-powerset runs for every combination of features - name: cargo hack run: cargo hack --feature-powerset check msrv: + # check that we can build using the minimal rust version that is specified by this crate runs-on: ubuntu-latest # we use a matrix here just because env can't be used in job names # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml index 00362a6..93a1698 100644 --- a/.github/workflows/nostd.yml +++ b/.github/workflows/nostd.yml @@ -1,10 +1,12 @@ +# This workflow checks whether the library is able to run without the std library (e.g., embedded). +# This entire file should be removed if this crate does not support no-std. See check.yml for +# information about how the concurrency cancelation and workflow triggering works permissions: contents: read on: push: branches: [main] pull_request: -# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index ce1e38e..afea8df 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -1,10 +1,16 @@ +# This workflow runs checks for unsafe code. In crates that don't have any unsafe code, this can be +# removed. Runs: +# - miri - detects undefined behavior and memory leaks +# - address santizer - detects memory errors +# - leak sanitizer - detects memory leaks +# - loom - Permutation testing for concurrent code https://crates.io/crates/loom +# See check.yml for information about how the concurrency cancelation and workflow triggering works permissions: contents: read on: push: branches: [main] pull_request: -# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 4607f3e..15b1a7c 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1,3 +1,6 @@ +# Run scheduled (rolling) jobs on a nightly basis, as your crate may break independently of any +# given PR. E.g., updates to rust nightly and updates to this crates dependencies. See check.yml for +# information about how the concurrency cancelation and workflow triggering works permissions: contents: read on: @@ -6,7 +9,6 @@ on: pull_request: schedule: - cron: '7 7 * * *' -# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -29,12 +31,16 @@ jobs: run: cargo test --locked --all-features --all-targets # https://twitter.com/alcuadrado/status/1571291687837732873 update: + # This action checks that updating the dependencies of this crate to the latest available that + # satisfy the versions in Cargo.toml does not break this crate. This is important as consumers + # of this crate will generally use the latest available crates. This is subject to the standard + # Cargo semver rules (i.e cargo does not update to a new major version unless explicitly told + # to). runs-on: ubuntu-latest name: ubuntu / beta / updated - # There's no point running this if no Cargo.lock was checked in in the - # first place, since we'd just redo what happened in the regular test job. - # Unfortunately, hashFiles only works in if on steps, so we reepeat it. - # if: hashFiles('Cargo.lock') != '' + # There's no point running this if no Cargo.lock was checked in in the first place, since we'd + # just redo what happened in the regular test job. Unfortunately, hashFiles only works in if on + # steps, so we repeat it. steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 56bd560..ab86e2d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,17 @@ +# This is the main CI workflow that runs the test suite on all pushes to main and all pull requests. +# It runs the following jobs: +# - required: runs the test suite on ubuntu with stable and beta rust toolchains +# - minimal: runs the test suite with the minimal versions of the dependencies that satisfy the +# requirements of this crate, and its dependencies +# - os-check: runs the test suite on mac and windows +# - coverage: runs the test suite and collects coverage information +# See check.yml for information about how the concurrency cancelation and workflow triggering works permissions: contents: read on: push: branches: [main] pull_request: -# Spend CI time only on latest ref: https://github.com/jonhoo/rust-ci-conf/pull/5 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -15,6 +22,8 @@ jobs: name: ubuntu / ${{ matrix.toolchain }} strategy: matrix: + # run on stable and beta to ensure that tests won't break on the next version of the rust + # toolchain toolchain: [stable, beta] steps: - uses: actions/checkout@v4 @@ -25,6 +34,7 @@ jobs: with: toolchain: ${{ matrix.toolchain }} - name: cargo generate-lockfile + # enable this ci template to run regardless of whether the lockfile is checked in or not if: hashFiles('Cargo.lock') == '' run: cargo generate-lockfile # https://twitter.com/jonhoo/status/1571290371124260865 @@ -34,6 +44,28 @@ jobs: - name: cargo test --doc run: cargo test --locked --all-features --doc minimal: + # This action chooses the oldest version of the dependencies permitted by Cargo.toml to ensure + # that this crate is compatible with the minimal version that this crate and its dependencies + # require. This will pickup issues where this create relies on functionality that was introduced + # later than the actual version specified (e.g., when we choose just a major version, but a + # method was added after this version). + # + # This particular check can be difficult to get to succeed as often transitive dependencies may + # be incorrectly specified (e.g., a dependency specifies 1.0 but really requires 1.1.5). There + # is an alternative flag available -Zminimal-direct that uses the minimal versions for direct + # dependencies of this crate, while selecting the maximal versions for the transitive + # dependencies. Alternatively, you can add a line in your Cargo.toml to artificially increase + # the minimal dependency, which you do with e.g.: + # ```toml + # # for minimal-versions + # [target.'cfg(any())'.dependencies] + # openssl = { version = "0.10.55", optional = true } # needed to allow foo to build with -Zminimal-versions + # ``` + # The optional = true is necessary in case that dependency isn't otherwise transitively required + # by your library, and the target bit is so that this dependency edge never actually affects + # Cargo build order. See also + # https://github.com/jonhoo/fantoccini/blob/fde336472b712bc7ebf5b4e772023a7ba71b2262/Cargo.toml#L47-L49. + # This action is run on ubuntu with the stable toolchain, as it is not expected to fail runs-on: ubuntu-latest name: ubuntu / stable / minimal-versions steps: @@ -51,6 +83,7 @@ jobs: - name: cargo test run: cargo test --locked --all-features --all-targets os-check: + # run cargo test on mac and windows runs-on: ${{ matrix.os }} name: ${{ matrix.os }} / stable strategy: @@ -75,6 +108,8 @@ jobs: - name: cargo test run: cargo test --locked --all-features --all-targets coverage: + # use llvm-cov to build and collect coverage and outputs in a format that is compatible with + # codecov.io runs-on: ubuntu-latest name: ubuntu / stable / coverage steps: From 3d6ab95662ede7ae2898cdc7bd5b75668c7b8ca0 Mon Sep 17 00:00:00 2001 From: Mathias Pius Date: Sat, 11 Nov 2023 15:01:33 +0100 Subject: [PATCH 061/105] Nit: Selecting direct minimal versions flag is -Zdirect-minimal-versions (#16) --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab86e2d..59028e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,14 +52,14 @@ jobs: # # This particular check can be difficult to get to succeed as often transitive dependencies may # be incorrectly specified (e.g., a dependency specifies 1.0 but really requires 1.1.5). There - # is an alternative flag available -Zminimal-direct that uses the minimal versions for direct - # dependencies of this crate, while selecting the maximal versions for the transitive + # is an alternative flag available -Zdirect-minimal-versions that uses the minimal versions for + # direct dependencies of this crate, while selecting the maximal versions for the transitive # dependencies. Alternatively, you can add a line in your Cargo.toml to artificially increase # the minimal dependency, which you do with e.g.: # ```toml # # for minimal-versions # [target.'cfg(any())'.dependencies] - # openssl = { version = "0.10.55", optional = true } # needed to allow foo to build with -Zminimal-versions + # openssl = { version = "0.10.55", optional = true } # needed to allow foo to build with -Zminimal-versions # ``` # The optional = true is necessary in case that dependency isn't otherwise transitively required # by your library, and the target bit is so that this dependency edge never actually affects From 60fdfbb65055e7e852820a75d9cf80b88d020702 Mon Sep 17 00:00:00 2001 From: Anas Date: Sun, 17 Dec 2023 17:52:41 +0200 Subject: [PATCH 062/105] chore: fix typos (#17) --- .github/workflows/nostd.yml | 2 +- .github/workflows/safety.yml | 4 ++-- .github/workflows/scheduled.yml | 2 +- .github/workflows/test.yml | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml index 93a1698..c12227a 100644 --- a/.github/workflows/nostd.yml +++ b/.github/workflows/nostd.yml @@ -1,6 +1,6 @@ # This workflow checks whether the library is able to run without the std library (e.g., embedded). # This entire file should be removed if this crate does not support no-std. See check.yml for -# information about how the concurrency cancelation and workflow triggering works +# information about how the concurrency cancellation and workflow triggering works permissions: contents: read on: diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml index afea8df..6bdd055 100644 --- a/.github/workflows/safety.yml +++ b/.github/workflows/safety.yml @@ -1,10 +1,10 @@ # This workflow runs checks for unsafe code. In crates that don't have any unsafe code, this can be # removed. Runs: # - miri - detects undefined behavior and memory leaks -# - address santizer - detects memory errors +# - address sanitizer - detects memory errors # - leak sanitizer - detects memory leaks # - loom - Permutation testing for concurrent code https://crates.io/crates/loom -# See check.yml for information about how the concurrency cancelation and workflow triggering works +# See check.yml for information about how the concurrency cancellation and workflow triggering works permissions: contents: read on: diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 15b1a7c..02aa275 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1,6 +1,6 @@ # Run scheduled (rolling) jobs on a nightly basis, as your crate may break independently of any # given PR. E.g., updates to rust nightly and updates to this crates dependencies. See check.yml for -# information about how the concurrency cancelation and workflow triggering works +# information about how the concurrency cancellation and workflow triggering works permissions: contents: read on: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59028e1..b98103d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ # requirements of this crate, and its dependencies # - os-check: runs the test suite on mac and windows # - coverage: runs the test suite and collects coverage information -# See check.yml for information about how the concurrency cancelation and workflow triggering works +# See check.yml for information about how the concurrency cancellation and workflow triggering works permissions: contents: read on: @@ -91,8 +91,8 @@ jobs: matrix: os: [macos-latest, windows-latest] steps: - # if your project needs OpenSSL, uncommment this to fix Windows builds. - # it's commented out by default as tthe install command takes 5-10m. + # if your project needs OpenSSL, uncomment this to fix Windows builds. + # it's commented out by default as the install command takes 5-10m. # - run: echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append # if: runner.os == 'Windows' # - run: vcpkg install openssl:x64-windows-static-md From a13691528b3f1918594bfc53db8cf5ad0240c59e Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sun, 17 Dec 2023 18:15:06 +0100 Subject: [PATCH 063/105] Remove stray trailing whitespace --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b98103d..fce7118 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,8 +57,8 @@ jobs: # dependencies. Alternatively, you can add a line in your Cargo.toml to artificially increase # the minimal dependency, which you do with e.g.: # ```toml - # # for minimal-versions - # [target.'cfg(any())'.dependencies] + # # for minimal-versions + # [target.'cfg(any())'.dependencies] # openssl = { version = "0.10.55", optional = true } # needed to allow foo to build with -Zminimal-versions # ``` # The optional = true is necessary in case that dependency isn't otherwise transitively required From f505e09b2a145de0df7445ca0ebe1f98b52ab3bc Mon Sep 17 00:00:00 2001 From: rtkay123 <70331483+rtkay123@users.noreply.github.com> Date: Sat, 20 Jan 2024 11:10:51 +0200 Subject: [PATCH 064/105] replace actions-rs/clippy-check with giraffate/clippy-action (#19) Co-authored-by: rtkay123 --- .github/workflows/check.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6b4d13a..c869ee7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -55,9 +55,10 @@ jobs: toolchain: ${{ matrix.toolchain }} components: clippy - name: cargo clippy - uses: actions-rs/clippy-check@v1 + uses: giraffate/clippy-action@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} + reporter: 'github-pr-check' + github_token: ${{ secrets.GITHUB_TOKEN }} doc: # run docs generation on nightly rather than stable. This enables features like # https://doc.rust-lang.org/beta/unstable-book/language-features/doc-cfg.html which allows an From 6f4092b0fe0e623f38589b6c325c0e4be2a136d1 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Thu, 1 Feb 2024 21:56:45 +0000 Subject: [PATCH 065/105] added some basic newer implementations --- Cargo.toml | 10 ++- src/lib.rs | 193 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 117 insertions(+), 86 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e3c402d..5e99a52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,12 @@ keywords = ["files", "utility", "filesystem"] categories = ["filesystem"] readme = "README.md" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] +anyhow = "1.0.79" chrono = "0.3" -uuid = { version = "0.8.1", features = ["v4"]} \ No newline at end of file +tokio = { version = "1.35.1", features = ["fs"] } +uuid = { version = "0.8.1", features = ["v4"]} + +[dev-dependencies] +tempfile = "3.9.0" +tokio = { version = "1.35.1", features = ["macros", "rt"] } diff --git a/src/lib.rs b/src/lib.rs index 77b3255..c7e2faa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,104 +1,131 @@ -//! Simple functions to help with file naming and file iteration -//! Ported from the [filetools](https://github.com/jgrizou/filetools) library written by Jonathan Grizou -//! -//! # Examples -//! -//! ``` -//! use filetools::filehelpers; -//! use std::path::PathBuf; -//! -//! fn main() -> Result <(), Box> { -//! /// Creating a directory -//! let new_path = PathBuf::from("./test"); -//! let _ = filehelpers::ensure_dir(new_path)?; -//! -//! /// Iterating through all files in a directory -//! let nr_search = PathBuf::from("./test"); -//! let r_search = PathBuf::from("./test"); -//! -//! // Non-recursive search of directroy, just files in search folder -//! let non_recursed_files = filehelpers::list_files(nr_search, false); -//! -//! // Recursive search of directory, gets all files in directory and all sub-directories -//! let recursed_files = filehelpers::list_files(r_search, true); -//! -//! /// Iterating through all folders in a directory -//! let nr_search = PathBuf::from("./test"); -//! let r_search = PathBuf::from("./test"); -//! -//! // Non-recursive search for all folders, just folders in search directory -//! let non_recursive_folders = filehelpers::list_folders(nr_search, false); -//! -//! // Recursive search of all folders, all subfolders in a directory as well -//! let recursive_folders = filehelpers::list_folders(r_search, true); -//! -//! Ok(()) -//! } -//! ``` -//! - -pub mod filenaming; -pub mod filehelpers; +use anyhow::{Context, Result}; +use std::{ + env, + path::{Component, Path}, +}; +use tokio::fs; + +// pub mod filehelpers; +// pub mod filenaming; + +// What do we want? +// +// Iterate files in a directory +// Iterate files in a directory +// +// Iterate folders in a directory + all subdirs +// Iterate folders in a directory + all subdirs +// +// Create a directory if not exists +// +// Check if a directory is a subdirectory +// +// Pattern match on a path +// +// All of the above but sync (feature) +// +// Naming patterns +// + +pub async fn create_directory(dir: impl AsRef) -> Result<()> { + fs::create_dir_all(dir) + .await + .context("unable to create directory")?; + + Ok(()) +} + +pub async fn is_subdir(path: impl AsRef, dir: impl AsRef) -> Result { + for component in path.as_ref().components() { + match component { + Component::Normal(p) => { + if p == dir.as_ref().as_os_str() { + return Ok(true); + } + } + _ => {} + } + } + + Ok(false) +} +fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe */) -> bool { + if let Some(p) = path.as_ref().to_str() { + if let Some(pat) = pattern.as_ref().to_str() { + return p.contains(&pat); + } + } + + false +} #[cfg(test)] mod tests { - use crate::{filehelpers, filenaming}; - use std::path::PathBuf; - - #[test] - fn iterate_files_and_folders() -> Result<(), Box> { - let files = filehelpers::list_files(PathBuf::from("src"), true)?; - let folders = filehelpers::list_folders(PathBuf::from("."), false)?; + use super::*; + use anyhow::{Context, Result}; - // filehelpers.rs filenaming.rs lib.rs - assert_eq!(files.len(), 3); + // This is needed as the `tempfile` lib doesn't like nested temp dirs + async fn create_tmpdir(path: &str) -> Result> { + let target = std::env::temp_dir().join(path); + tokio::fs::create_dir_all(&target) + .await + .context("creating tempdir")?; - // target/ src/ .git/ - assert_eq!(folders.len(), 4); - Ok(()) + Ok(target) } - #[test] - fn folder_creation() { - let _ = filehelpers::ensure_dir(PathBuf::from("./test/func")); - } + #[tokio::test] + // This is kind of redundant as it just wraps `tokio::fs::create_dir_all` + // but yay for test coverage i suppose + async fn creates_a_directory() -> Result<()> { + let tmp = std::env::temp_dir(); - #[test] - fn generate_filenames() -> Result<(), Box> { - let name1 = filenaming::generate_default_timestamped_name("", ".pdf"); - let name2 = filenaming::generate_default_timestamped_name("test_file", ".dxf"); - let name3 = filenaming::generate_random_name(".docx"); - let name4 = filenaming::generate_n_digit_name(55, 6, ".pdf"); + // Creates a single directory + let single_path = tmp.join("create_dir"); + create_directory(&single_path) + .await + .context("create directory single")?; + + assert!(single_path.exists()); - println!("Name1: {:?}", name1); - println!("Name2: {:?}", name2); - println!("Name3: {:?}", name3); - println!("Name4: {:?}", name4); + // Nested directories + let nested_path = tmp.join("create_dir/test/this/is/nested"); + create_directory(&nested_path) + .await + .context("create directory nested")?; - assert_eq!(name4, PathBuf::from("000055.pdf")); + assert!(nested_path.exists()); Ok(()) } - #[test] - fn path_does_contains() -> Result<(), Box> { - let path1 = PathBuf::from("./target/doc/cfg_if"); - let path2 = PathBuf::from("./target/chrono/datetime"); - let path3 = PathBuf::from("./target"); + #[tokio::test] + async fn checks_if_a_directory_is_a_subdirectory() -> Result<()> { + let path = create_tmpdir("is_subdir/this/is/a/nested/tmp/dir") + .await + .context("creating nested tempdirs")?; - let target_paths: Vec = filehelpers::list_files(path3, true)? - .into_iter() - .filter(|x| filehelpers::path_contains(x.to_path_buf(), "doc")) - .collect(); + let mut result = is_subdir(&path, "nested").await.context("is_subdir test")?; + assert!(result); - assert_eq!(filehelpers::path_contains(path1, "doc"), true); - assert_eq!(filehelpers::path_contains(path2, "debug"), false); - - for path in target_paths.iter() { - assert_eq!(filehelpers::path_contains(path.to_path_buf(), "doc"), true); - } + result = is_subdir(&path, "not_valid") + .await + .context("is_subdir test")?; + assert!(!result); Ok(()) } + + #[test] + fn check_path_contains_subpath() { + let main = "I/am/a/path/hello/there"; + assert!(path_contains(main, "a/path")); + assert!(!path_contains(main, "not")); + + // Check it works for paths + let main = Path::new(main); + assert!(path_contains(main, Path::new("a/path"))); + assert!(!path_contains(main, Path::new("not"))); + } } From 5ea8dd474f76850d356bf9e36815d2cec17c42ad Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Thu, 1 Feb 2024 21:59:10 +0000 Subject: [PATCH 066/105] note update --- src/lib.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c7e2faa..0843235 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,5 @@ use anyhow::{Context, Result}; -use std::{ - env, - path::{Component, Path}, -}; +use std::path::{Component, Path}; use tokio::fs; // pub mod filehelpers; @@ -11,16 +8,16 @@ use tokio::fs; // What do we want? // // Iterate files in a directory -// Iterate files in a directory +// Iterate folders in a directory // -// Iterate folders in a directory + all subdirs +// Iterate files in a directory + all subdirs // Iterate folders in a directory + all subdirs // -// Create a directory if not exists +// Create a directory if not exists - DONE // -// Check if a directory is a subdirectory +// Check if a directory is a subdirectory - DONE // -// Pattern match on a path +// Pattern match on a path - DONE // // All of the above but sync (feature) // @@ -50,7 +47,7 @@ pub async fn is_subdir(path: impl AsRef, dir: impl AsRef) -> Result< Ok(false) } -fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe */) -> bool { +pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe */) -> bool { if let Some(p) = path.as_ref().to_str() { if let Some(pat) = pattern.as_ref().to_str() { return p.contains(&pat); From ab0be9cd651354777d6b9f4bc2ee1a5d5d33d913 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 2 Feb 2024 18:03:30 +0000 Subject: [PATCH 067/105] list_files added --- Cargo.toml | 1 + src/lib.rs | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5e99a52..996b755 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ readme = "README.md" [dependencies] anyhow = "1.0.79" +async-recursion = "1.0.5" chrono = "0.3" tokio = { version = "1.35.1", features = ["fs"] } uuid = { version = "0.8.1", features = ["v4"]} diff --git a/src/lib.rs b/src/lib.rs index 0843235..87b09d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ use anyhow::{Context, Result}; -use std::path::{Component, Path}; +use async_recursion::async_recursion; +use std::path::{Component, Path, PathBuf}; use tokio::fs; // pub mod filehelpers; @@ -57,10 +58,64 @@ pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe false } +pub async fn list_files(path: impl AsRef) -> Result>> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + + let mut files = vec![]; + if path.as_ref().is_file() { + return Ok(files); + } + + let mut entries = fs::read_dir(path.as_ref()) + .await + .context("list_files directory read")?; + + while let Some(entry) = entries.next_entry().await? { + let e_path = entry.path(); + + if e_path.is_file() { + files.push(e_path); + } + } + + Ok(files) +} + +#[async_recursion] +pub async fn list_files_recursive + Send>(path: P) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + + let mut files = vec![]; + if path.as_ref().is_file() { + return Ok(files); + } + + let mut entries = fs::read_dir(path.as_ref()) + .await + .context("list_files directory read")?; + + while let Some(entry) = entries.next_entry().await? { + let e_path = entry.path(); + + if e_path.is_file() { + files.push(e_path); + } else if e_path.is_dir() { + files.extend( + list_files_recursive(e_path) + .await + .context("recursive list_files call")?, + ); + } + } + + Ok(files) +} + #[cfg(test)] mod tests { use super::*; use anyhow::{Context, Result}; + use std::path::PathBuf; // This is needed as the `tempfile` lib doesn't like nested temp dirs async fn create_tmpdir(path: &str) -> Result> { @@ -116,6 +171,7 @@ mod tests { #[test] fn check_path_contains_subpath() { + // Basic str let main = "I/am/a/path/hello/there"; assert!(path_contains(main, "a/path")); assert!(!path_contains(main, "not")); @@ -124,5 +180,76 @@ mod tests { let main = Path::new(main); assert!(path_contains(main, Path::new("a/path"))); assert!(!path_contains(main, Path::new("not"))); + + // Pathbufs? + let main = PathBuf::from("I/am/a/path/hello/there"); + assert!(path_contains(&main, PathBuf::from("a/path"))); + assert!(!path_contains(main, PathBuf::from("not"))); + + // What about strings? + assert!(path_contains( + String::from("I/am/a/path/hello/there"), + String::from("a/path") + )); + assert!(!path_contains( + String::from("I/am/a/path/hello/there"), + String::from("not") + )); + } + + #[tokio::test] + async fn check_list_files_works() -> Result<()> { + let tmp = std::env::temp_dir(); + let folder = tmp.join("list_files_example"); + create_directory(&folder).await?; + + tokio::fs::File::create(&folder.join("first.rs")).await?; + tokio::fs::File::create(&folder.join("second.txt")).await?; + tokio::fs::File::create(&folder.join("another_second.php")).await?; + tokio::fs::File::create(&folder.join("third.c")).await?; + + let res = list_files(&folder).await?; + assert_eq!(res.len(), 4); + + // Errors on non-exist + assert!( + list_files_recursive("iDoNotExistAndNeVerWillUnlessYouIntenTionallyCreateMe") + .await + .is_err() + ); + + tokio::fs::remove_dir_all(&folder).await?; + + Ok(()) + } + + #[tokio::test] + async fn check_list_files_recursive_works() -> Result<()> { + let tmp = std::env::temp_dir(); + let ffolder = tmp.join("first"); + let sfolder = ffolder.join("second"); + let tfolder = sfolder.join("third"); + create_directory(&ffolder).await?; + create_directory(&sfolder).await?; + create_directory(&tfolder).await?; + + tokio::fs::File::create(ffolder.join("first.rs")).await?; + tokio::fs::File::create(sfolder.join("second.txt")).await?; + tokio::fs::File::create(sfolder.join("another_second.php")).await?; + tokio::fs::File::create(tfolder.join("third.c")).await?; + + let res = list_files_recursive(&ffolder).await?; + assert_eq!(res.len(), 4); + + // Errors on an non-existant path + assert!( + list_files_recursive("iDoNotExistAndNeVerWillUnlessYouIntenTionallyCreateMe") + .await + .is_err() + ); + + tokio::fs::remove_dir_all(ffolder).await?; + + Ok(()) } } From 8f6c69b83205176e4673c29ad8c6da92511a4a64 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 2 Feb 2024 18:33:14 +0000 Subject: [PATCH 068/105] working on a tempfile test helper --- Cargo.toml | 1 - src/lib.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 996b755..7eca3c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,4 @@ tokio = { version = "1.35.1", features = ["fs"] } uuid = { version = "0.8.1", features = ["v4"]} [dev-dependencies] -tempfile = "3.9.0" tokio = { version = "1.35.1", features = ["macros", "rt"] } diff --git a/src/lib.rs b/src/lib.rs index 87b09d5..1cca61d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -117,6 +117,66 @@ mod tests { use anyhow::{Context, Result}; use std::path::PathBuf; + /// Helper for creating temp directories + /// + /// Tempfile _would_ work but I want nested dirs and easy ways to create + /// a series of files / folder quickly without worrying + struct TempPath { + root: PathBuf, + pub path: PathBuf, + } + + impl TempPath { + pub async fn new(p: impl AsRef) -> Result { + let root = std::env::temp_dir(); + let path = if p.as_ref().starts_with(&root) { + p.as_ref().to_path_buf() + } else { + root.join(p) + }; + + create_directory(&path).await?; + + Ok(Self { root, path }) + } + + pub async fn new_file(&self, name: impl AsRef) -> Result { + let p = self.path.join(name); + tokio::fs::File::create(&p).await?; + + Self::new(p).await + } + + pub async fn multi_files(&self, names: Vec>) -> Result<()> { + for name in names { + tokio::fs::File::create(&self.path.join(name)).await?; + } + + Ok(()) + } + + pub async fn new_folder(&self, name: impl AsRef) -> Result { + let p = self.path.join(name); + create_directory(&p).await?; + + Self::new(p).await + } + + pub async fn multi_folder(&self, names: Vec>) -> Result<()> { + for name in names { + create_directory(&self.path.join(name)).await?; + } + + Ok(()) + } + + pub async fn cleanup(self) -> Result<()> { + tokio::fs::remove_dir_all(&self.path).await?; + drop(self); + Ok(()) + } + } + // This is needed as the `tempfile` lib doesn't like nested temp dirs async fn create_tmpdir(path: &str) -> Result> { let target = std::env::temp_dir().join(path); @@ -252,4 +312,11 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn testfile_struct() -> Result<()> { + // TODO: Test this temp path thing actually works + let tmp = TempPath::new("testpath"); + Ok(()) + } } From f8f04aefdf3c756e6a769399ce2ac89979338ad0 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 3 Feb 2024 09:31:12 +0100 Subject: [PATCH 069/105] Semi-breaking: update codecov action Note: this requires adding `CODECOV_TOKEN` to your GitHub repository's secrets! See associated comment in the commit content. --- .github/workflows/test.yml | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fce7118..debdfe8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,8 +108,27 @@ jobs: - name: cargo test run: cargo test --locked --all-features --all-targets coverage: - # use llvm-cov to build and collect coverage and outputs in a format that is compatible with - # codecov.io + # use llvm-cov to build and collect coverage and outputs in a format that + # is compatible with codecov.io + # + # note that codecov as of v4 requires that CODECOV_TOKEN from + # + # https://app.codecov.io/gh///settings + # + # is set in two places on your repo: + # + # - https://github.com/jonhoo/guardian/settings/secrets/actions + # - https://github.com/jonhoo/guardian/settings/secrets/dependabot + # + # (the former is needed for codecov uploads to work with Dependabot PRs) + # + # PRs coming from forks of your repo will not have access to the token, but + # for those, codecov allows uploading coverage reports without a token. + # it's all a little weird and inconvenient. see + # + # https://github.com/codecov/feedback/issues/112 + # + # for lots of more discussion runs-on: ubuntu-latest name: ubuntu / stable / coverage steps: @@ -127,7 +146,11 @@ jobs: run: cargo generate-lockfile - name: cargo llvm-cov run: cargo llvm-cov --locked --all-features --lcov --output-path lcov.info + - name: record Rust version + run: echo "RUST=$(rustc --version)" >> "$GITHUB_ENV" - name: Upload to codecov.io - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + env_vars: OS,RUST From caa3616c250361bac585b6979e5390e403b45097 Mon Sep 17 00:00:00 2001 From: Jon Gjengset Date: Sat, 3 Feb 2024 09:44:41 +0100 Subject: [PATCH 070/105] Uniform capitalization --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index debdfe8..f7540ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -146,7 +146,7 @@ jobs: run: cargo generate-lockfile - name: cargo llvm-cov run: cargo llvm-cov --locked --all-features --lcov --output-path lcov.info - - name: record Rust version + - name: Record Rust version run: echo "RUST=$(rustc --version)" >> "$GITHUB_ENV" - name: Upload to codecov.io uses: codecov/codecov-action@v4 From 2dc83f560b5f60684a5c08afed6a4ac6c88dcada Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 3 Feb 2024 14:04:23 +0000 Subject: [PATCH 071/105] updated tests using new testing helper --- src/lib.rs | 90 ++++++++++++++++++++++-------------------------------- 1 file changed, 37 insertions(+), 53 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1cca61d..a07f7c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,10 +8,10 @@ use tokio::fs; // What do we want? // -// Iterate files in a directory +// Iterate files in a directory - DONE // Iterate folders in a directory // -// Iterate files in a directory + all subdirs +// Iterate files in a directory + all subdirs - DONE // Iterate folders in a directory + all subdirs // // Create a directory if not exists - DONE @@ -26,9 +26,11 @@ use tokio::fs; // pub async fn create_directory(dir: impl AsRef) -> Result<()> { - fs::create_dir_all(dir) - .await - .context("unable to create directory")?; + if !dir.as_ref().exists() { + fs::create_dir_all(dir) + .await + .context("unable to create directory")?; + } Ok(()) } @@ -92,7 +94,7 @@ pub async fn list_files_recursive + Send>(path: P) -> Result) -> Result { @@ -147,7 +148,7 @@ mod tests { Self::new(p).await } - pub async fn multi_files(&self, names: Vec>) -> Result<()> { + pub async fn multi_file(&self, names: Vec>) -> Result<()> { for name in names { tokio::fs::File::create(&self.path.join(name)).await?; } @@ -170,6 +171,10 @@ mod tests { Ok(()) } + pub fn join(&self, path: impl AsRef) -> impl AsRef { + self.path.join(path) + } + pub async fn cleanup(self) -> Result<()> { tokio::fs::remove_dir_all(&self.path).await?; drop(self); @@ -259,64 +264,43 @@ mod tests { #[tokio::test] async fn check_list_files_works() -> Result<()> { - let tmp = std::env::temp_dir(); - let folder = tmp.join("list_files_example"); - create_directory(&folder).await?; - - tokio::fs::File::create(&folder.join("first.rs")).await?; - tokio::fs::File::create(&folder.join("second.txt")).await?; - tokio::fs::File::create(&folder.join("another_second.php")).await?; - tokio::fs::File::create(&folder.join("third.c")).await?; + let root = TempPath::new("lf_test").await?; + root.multi_file(vec!["first.rs", "second.c", "third.js", "fourth.rb"]) + .await?; - let res = list_files(&folder).await?; + let res = list_files(root.path.clone()).await?; assert_eq!(res.len(), 4); - // Errors on non-exist - assert!( - list_files_recursive("iDoNotExistAndNeVerWillUnlessYouIntenTionallyCreateMe") - .await - .is_err() - ); + assert!(list_files("IDoNotExistAsADirectoryOrShouldntAtLeAst") + .await + .is_err()); - tokio::fs::remove_dir_all(&folder).await?; + root.cleanup().await?; Ok(()) } #[tokio::test] async fn check_list_files_recursive_works() -> Result<()> { - let tmp = std::env::temp_dir(); - let ffolder = tmp.join("first"); - let sfolder = ffolder.join("second"); - let tfolder = sfolder.join("third"); - create_directory(&ffolder).await?; - create_directory(&sfolder).await?; - create_directory(&tfolder).await?; - - tokio::fs::File::create(ffolder.join("first.rs")).await?; - tokio::fs::File::create(sfolder.join("second.txt")).await?; - tokio::fs::File::create(sfolder.join("another_second.php")).await?; - tokio::fs::File::create(tfolder.join("third.c")).await?; - - let res = list_files_recursive(&ffolder).await?; - assert_eq!(res.len(), 4); + let root = TempPath::new("lfr_test").await?; + let ffolder = root.new_folder("ffolder").await?; + let sfolder = root.new_folder("sfolder").await?; + let tfolder = root.new_folder("tfolder").await?; - // Errors on an non-existant path - assert!( - list_files_recursive("iDoNotExistAndNeVerWillUnlessYouIntenTionallyCreateMe") - .await - .is_err() - ); + root.new_file("initial.pdf").await?; + ffolder.new_file("first.rs").await?; + sfolder.multi_file(vec!["second.txt", "third.php"]).await?; + tfolder.new_file("fourth.cpp").await?; - tokio::fs::remove_dir_all(ffolder).await?; + let res = list_files_recursive(&root.path).await?; + assert_eq!(res.len(), 5); - Ok(()) - } + assert!(list_files("IDoNotExistAsADirectoryOrShouldntAtLeAst") + .await + .is_err()); + + root.cleanup().await?; - #[tokio::test] - async fn testfile_struct() -> Result<()> { - // TODO: Test this temp path thing actually works - let tmp = TempPath::new("testpath"); Ok(()) } } From fbb7102032ef36b5212b6b4f5f7078c266ccd2c6 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 3 Feb 2024 15:09:21 +0000 Subject: [PATCH 072/105] working on list_folders --- src/lib.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index a07f7c4..5de4fbc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -83,6 +83,23 @@ pub async fn list_files(path: impl AsRef) -> Result>> Ok(files) } +pub async fn list_folders(path: impl AsRef) -> Result>> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + + let mut folders = vec![]; + let mut entries = fs::read_dir(path.as_ref()) + .await + .context("list_folders direcrory read")?; + + while let Some(entry) = entries.next_entry().await? { + let e_path = entry.path(); + if e_path.is_dir() { + folders.push(e_path); + } + } + Ok(folders) +} + #[async_recursion] pub async fn list_files_recursive + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); @@ -113,6 +130,31 @@ pub async fn list_files_recursive + Send>(path: P) -> Result + Send>(path: P) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + + let mut folders = vec![]; + let mut entries = fs::read_dir(path.as_ref()) + .await + .context("list_folders_recursive directory read")?; + + while let Some(entry) = entries.next_entry().await? { + let e_path = entry.path(); + + if e_path.is_dir() { + folders.push(e_path); + folders.extend( + list_folders_recursive(e_path) + .await + .context("list_folders_recursive inner call"), + ); + } + } + + Ok(folders) +} + #[cfg(test)] mod tests { use super::*; From 5bb7fe66f869b06033efe5435d12b3941f605bc2 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 3 Feb 2024 16:09:02 +0000 Subject: [PATCH 073/105] list_folders and list_folders_recursive complete --- src/lib.rs | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5de4fbc..d4bd5c6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,10 +9,10 @@ use tokio::fs; // What do we want? // // Iterate files in a directory - DONE -// Iterate folders in a directory +// Iterate folders in a directory - DONE // // Iterate files in a directory + all subdirs - DONE -// Iterate folders in a directory + all subdirs +// Iterate folders in a directory + all subdirs - DONE // // Create a directory if not exists - DONE // @@ -143,11 +143,11 @@ pub async fn list_folders_recursive + Send>(path: P) -> Result Result<()> { + let root = TempPath::new("lfolder_test").await?; + root.multi_folder(vec!["folder1", "folder2", "folder3", "folder4"]) + .await?; + + let res = list_folders(root.path.clone()).await?; + assert_eq!(res.len(), 4); + + assert!(list_folders("non-existant_path").await.is_err()); + + root.cleanup().await?; + Ok(()) + } + + #[tokio::test] + async fn check_list_folders_recursive_works() -> Result<()> { + let root = TempPath::new("lfolderrec_test").await?; + root.multi_folder(vec!["folder1", "folder2"]).await?; + + let f1 = TempPath::new(root.join("folder1")).await?; + f1.multi_folder(vec!["sub1", "sub2", "sub3"]).await?; + + let s2 = TempPath::new(f1.join("sub2")).await?; + s2.multi_folder(vec!["deep1", "deep2"]).await?; + + let res = list_folders_recursive(root.path.clone()).await?; + assert_eq!(res.len(), 7); + + assert!(list_folders_recursive("not-a-valId_pathd").await.is_err()); + + root.cleanup().await?; + + Ok(()) + } } From acb4920fae102cc9c88e3909b4cb211903ec874f Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 3 Feb 2024 16:50:02 +0000 Subject: [PATCH 074/105] refactored listing files and folders --- src/lib.rs | 126 ++++++++++++++++++++++++----------------------------- 1 file changed, 58 insertions(+), 68 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d4bd5c6..f960f80 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,8 @@ use async_recursion::async_recursion; use std::path::{Component, Path, PathBuf}; use tokio::fs; +pub mod naming; + // pub mod filehelpers; // pub mod filenaming; @@ -25,6 +27,14 @@ use tokio::fs; // Naming patterns // +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum FtIterItemState { + FileNoRec, + FileRec, + DirNoRec, + DirRec, +} + pub async fn create_directory(dir: impl AsRef) -> Result<()> { if !dir.as_ref().exists() { fs::create_dir_all(dir) @@ -60,99 +70,79 @@ pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe false } -pub async fn list_files(path: impl AsRef) -> Result>> { +pub async fn list_files + Send>(path: P) -> Result>> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); - let mut files = vec![]; - if path.as_ref().is_file() { - return Ok(files); - } - - let mut entries = fs::read_dir(path.as_ref()) - .await - .context("list_files directory read")?; - - while let Some(entry) = entries.next_entry().await? { - let e_path = entry.path(); - - if e_path.is_file() { - files.push(e_path); - } - } - - Ok(files) + iteritems(path, FtIterItemState::FileNoRec).await } -pub async fn list_folders(path: impl AsRef) -> Result>> { +pub async fn list_folders + Send>(path: P) -> Result>> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); - - let mut folders = vec![]; - let mut entries = fs::read_dir(path.as_ref()) - .await - .context("list_folders direcrory read")?; - - while let Some(entry) = entries.next_entry().await? { - let e_path = entry.path(); - if e_path.is_dir() { - folders.push(e_path); - } - } - Ok(folders) + iteritems(path, FtIterItemState::DirNoRec).await } #[async_recursion] pub async fn list_files_recursive + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); - let mut files = vec![]; - if path.as_ref().is_file() { - return Ok(files); - } - - let mut entries = fs::read_dir(path.as_ref()) - .await - .context("list_files_recursive directory read")?; - - while let Some(entry) = entries.next_entry().await? { - let e_path = entry.path(); - - if e_path.is_file() { - files.push(e_path); - } else if e_path.is_dir() { - files.extend( - list_files_recursive(e_path) - .await - .context("recursive list_files call")?, - ); - } - } - - Ok(files) + iteritems(path, FtIterItemState::FileRec).await } #[async_recursion] pub async fn list_folders_recursive + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + iteritems(path, FtIterItemState::DirRec).await +} + +#[async_recursion] +async fn iteritems + Send>( + path: P, + iterstate: FtIterItemState, +) -> Result> { + let mut items = vec![]; - let mut folders = vec![]; let mut entries = fs::read_dir(path.as_ref()) .await - .context("list_folders_recursive directory read")?; + .context("list items inner call")?; while let Some(entry) = entries.next_entry().await? { let e_path = entry.path(); - - if e_path.is_dir() { - folders.push(e_path.clone()); - folders.extend( - list_folders_recursive(e_path) - .await - .context("list_folders_recursive inner call")?, - ); + match iterstate { + FtIterItemState::FileNoRec => { + if e_path.is_file() { + items.push(e_path); + } + } + FtIterItemState::FileRec => { + if e_path.is_file() { + items.push(e_path) + } else if e_path.is_dir() { + items.extend(iteritems(e_path, iterstate).await?); + } + } + FtIterItemState::DirNoRec => { + if e_path.is_dir() { + items.push(e_path); + } + } + FtIterItemState::DirRec => { + if e_path.is_dir() { + items.push(e_path.clone()); + items.extend(iteritems(e_path, iterstate).await?); + } + } } } - Ok(folders) + Ok(items) } #[cfg(test)] From 778d656bdf6086ab497a9bcb32409491019d6805 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 3 Feb 2024 17:37:47 +0000 Subject: [PATCH 075/105] updated Test helpers and some tests --- src/lib.rs | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f960f80..bbdfae9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,8 +3,6 @@ use async_recursion::async_recursion; use std::path::{Component, Path, PathBuf}; use tokio::fs; -pub mod naming; - // pub mod filehelpers; // pub mod filenaming; @@ -203,25 +201,25 @@ mod tests { Ok(()) } - pub fn join(&self, path: impl AsRef) -> impl AsRef { - self.path.join(path) + pub async fn nest_folders(&self, subfolder_chain: Vec>) -> Result { + let mut dst_path = self.path.clone(); + for sf in subfolder_chain { + dst_path = dst_path.join(sf.as_ref()); + } + + create_directory(&dst_path).await?; + Self::new(dst_path).await } - pub async fn cleanup(self) -> Result<()> { - tokio::fs::remove_dir_all(&self.path).await?; - drop(self); - Ok(()) + pub fn join(&self, path: impl AsRef) -> impl AsRef { + self.path.join(path) } } - // This is needed as the `tempfile` lib doesn't like nested temp dirs - async fn create_tmpdir(path: &str) -> Result> { - let target = std::env::temp_dir().join(path); - tokio::fs::create_dir_all(&target) - .await - .context("creating tempdir")?; - - Ok(target) + impl Drop for TempPath { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } } #[tokio::test] @@ -246,19 +244,24 @@ mod tests { assert!(nested_path.exists()); + std::fs::remove_dir_all(single_path)?; + Ok(()) } #[tokio::test] async fn checks_if_a_directory_is_a_subdirectory() -> Result<()> { - let path = create_tmpdir("is_subdir/this/is/a/nested/tmp/dir") + let root = TempPath::new("is_subdir").await?; + let nested = root + .nest_folders(vec!["this", "is", "a", "nested", "tmp", "dir"]) + .await?; + let mut result = is_subdir(&nested.path, "nested") .await - .context("creating nested tempdirs")?; + .context("is_subdir test")?; - let mut result = is_subdir(&path, "nested").await.context("is_subdir test")?; assert!(result); - result = is_subdir(&path, "not_valid") + result = is_subdir(&nested.path, "not_valid") .await .context("is_subdir test")?; @@ -307,8 +310,6 @@ mod tests { .await .is_err()); - root.cleanup().await?; - Ok(()) } @@ -331,8 +332,6 @@ mod tests { .await .is_err()); - root.cleanup().await?; - Ok(()) } @@ -347,7 +346,6 @@ mod tests { assert!(list_folders("non-existant_path").await.is_err()); - root.cleanup().await?; Ok(()) } @@ -367,8 +365,6 @@ mod tests { assert!(list_folders_recursive("not-a-valId_pathd").await.is_err()); - root.cleanup().await?; - Ok(()) } } From a4db80e2ff1ac7f1a961736e0bfa8b6dd86f7bbd Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 3 Feb 2024 17:50:01 +0000 Subject: [PATCH 076/105] doc updates and some notes --- Cargo.toml | 8 ++++---- src/lib.rs | 27 ++++++++++----------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7eca3c8..e9a7bb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "filetools" -version = "0.2.0" +version = "0.3.0" authors = ["Graham Keenan "] -edition = "2018" -description = "Port of Jonathan Grizou's filetools Python library (https://github.com/jgrizou/filetools)" +edition = "2021" +description = "General filehelpers to make some file operations easier" repository = "https://github.com/Tyrannican/filetools-rs" -license = "MIT" +license = "MIT OR Apache-2.0" keywords = ["files", "utility", "filesystem"] categories = ["filesystem"] readme = "README.md" diff --git a/src/lib.rs b/src/lib.rs index bbdfae9..d8a94eb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,27 +3,19 @@ use async_recursion::async_recursion; use std::path::{Component, Path, PathBuf}; use tokio::fs; -// pub mod filehelpers; -// pub mod filenaming; - // What do we want? // -// Iterate files in a directory - DONE -// Iterate folders in a directory - DONE -// -// Iterate files in a directory + all subdirs - DONE -// Iterate folders in a directory + all subdirs - DONE -// -// Create a directory if not exists - DONE -// -// Check if a directory is a subdirectory - DONE -// -// Pattern match on a path - DONE -// -// All of the above but sync (feature) +// MUST: +// Naming helpers +// Sync functionality of the current operations +// Docs etc. // -// Naming patterns +// NICE: +// Some kind of filtering for searching? +// A higher level File type to perform some ops quicker? // +// MILES OFF: +// File Task Manager - That operation scheduler thing #[derive(Debug, Copy, Clone, PartialEq, Eq)] enum FtIterItemState { @@ -153,6 +145,7 @@ mod tests { /// /// Tempfile _would_ work but I want nested dirs and easy ways to create /// a series of files / folder quickly without worrying + /// A cheap knock-off of `Tempfile` but meh, this works kinda better for my use case struct TempPath { pub path: PathBuf, } From 20fcf2e33452b2b70382ba429b873a5cfc30e2c5 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sun, 4 Feb 2024 19:15:56 +0000 Subject: [PATCH 077/105] working on filenaming tests --- Cargo.toml | 1 + src/filenaming.rs | 79 -------------------------------- src/lib.rs | 16 ++++--- src/naming.rs | 112 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 86 deletions(-) delete mode 100644 src/filenaming.rs create mode 100644 src/naming.rs diff --git a/Cargo.toml b/Cargo.toml index e9a7bb7..cd8f7ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ uuid = { version = "0.8.1", features = ["v4"]} [dev-dependencies] tokio = { version = "1.35.1", features = ["macros", "rt"] } +regex = "1.10.3" diff --git a/src/filenaming.rs b/src/filenaming.rs deleted file mode 100644 index df0ac95..0000000 --- a/src/filenaming.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Functions that generate PathBuf filenames -//! -//! # Examples -//! -//! ``` -//! use std::path::PathBuf; -//! use filetools::filenaming; -//! -//! fn main() { -//! let custom_name = filenaming::generate_name("test", ".pdf"); -//! assert_eq!(custom_name, PathBuf::from("test.pdf")); -//! -//! // Name will be suffixed by the current time it was generated -//! let timestamped_name = filenaming::generate_default_timestamped_name("test", ".pdf"); -//! -//! // Random name is a UUIDv4 string suffixed by the extension -//! let random_name = filenaming::generate_random_name(".pdf"); -//! -//! // N-digit name is a number prefixed by X zeros -//! let n_digit_name = filenaming::generate_n_digit_name(5, 4, ".pdf"); -//! assert_eq!(n_digit_name, PathBuf::from("0005.pdf")); -//! } -//! ``` -//! - -use std::path::PathBuf; -use uuid::Uuid; -use chrono::prelude::*; - -/// Generates a `PathBuf` from a given and extension -/// -/// Returns a `PathBuf` of the form `name.ext` -pub fn generate_name(name: &str, ext: &str) -> PathBuf { - PathBuf::from(format!("{}{}", name, ext)) -} - -/// Generates a `PathBuf` from a name and extention with a default timestamp of "DD_MM_YY_HHMMSS" -/// If `fname` is "", just uses the timestamp and extension -/// -/// Returns `PathBuf` in the form `fname_timestamp.ext` -pub fn generate_default_timestamped_name(fname: &str, ext: &str) -> PathBuf { - let dt = UTC::now().format("%d_%m_%Y_%Hh%Mm%Ss"); - - if fname.len() == 0 { - PathBuf::from(format!("{}{}", dt, ext)) - } else { - PathBuf::from(format!("{}_{}{}", fname, dt, ext)) - } -} - -/// Generates a `PathBuf` from a name and extension with a given timestamp format -/// If `fname` is an empty string, returns just the timestamp suffixed with the extension. -/// -/// Returns `PathBuf` in the form `fname_timestamp.ext` -pub fn generate_timestamped_name(fname: &str, ext: &str, fmt: &str) -> PathBuf { - let dt = UTC::now().format(fmt); - - if fname.len() == 0 { - PathBuf::from(format!("{}{}", dt, ext)) - } else { - PathBuf::from(format!("{}_{}{}", fname, dt, ext)) - } -} - -/// Generates a random UUIDv4 `PathBuf` -/// -/// Returns `PathBuf` in the form `uuid.ext` -pub fn generate_random_name(ext: &str) -> PathBuf { - let unique = Uuid::new_v4(); - - PathBuf::from(format!("{}{}", unique.to_string(), ext)) -} - -/// Generates a `PathBuf` from a `number` prefixed by `n_digits` zeros -/// -/// Returns `PathBuf` of the form e.g `0005.ext` -pub fn generate_n_digit_name(number: i32, n_digits: usize, ext: &str) -> PathBuf { - PathBuf::from(format!("{:0fill$}{}", number, ext, fill=n_digits)) -} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index d8a94eb..980d82c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,8 @@ use async_recursion::async_recursion; use std::path::{Component, Path, PathBuf}; use tokio::fs; +pub mod naming; + // What do we want? // // MUST: @@ -25,7 +27,7 @@ enum FtIterItemState { DirRec, } -pub async fn create_directory(dir: impl AsRef) -> Result<()> { +pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { if !dir.as_ref().exists() { fs::create_dir_all(dir) .await @@ -159,7 +161,7 @@ mod tests { root.join(p) }; - create_directory(&path).await?; + ensure_directory(&path).await?; Ok(Self { path }) } @@ -181,14 +183,14 @@ mod tests { pub async fn new_folder(&self, name: impl AsRef) -> Result { let p = self.path.join(name); - create_directory(&p).await?; + ensure_directory(&p).await?; Self::new(p).await } pub async fn multi_folder(&self, names: Vec>) -> Result<()> { for name in names { - create_directory(&self.path.join(name)).await?; + ensure_directory(&self.path.join(name)).await?; } Ok(()) @@ -200,7 +202,7 @@ mod tests { dst_path = dst_path.join(sf.as_ref()); } - create_directory(&dst_path).await?; + ensure_directory(&dst_path).await?; Self::new(dst_path).await } @@ -223,7 +225,7 @@ mod tests { // Creates a single directory let single_path = tmp.join("create_dir"); - create_directory(&single_path) + ensure_directory(&single_path) .await .context("create directory single")?; @@ -231,7 +233,7 @@ mod tests { // Nested directories let nested_path = tmp.join("create_dir/test/this/is/nested"); - create_directory(&nested_path) + ensure_directory(&nested_path) .await .context("create directory nested")?; diff --git a/src/naming.rs b/src/naming.rs new file mode 100644 index 0000000..abf7f38 --- /dev/null +++ b/src/naming.rs @@ -0,0 +1,112 @@ +//! Functions that generate PathBuf filenames +//! +//! # Examples +//! +//! ``` +//! use std::path::PathBuf; +//! use filetools::naming; +//! +//! fn main() { +//! let custom_name = naming::generate_name("test", "pdf"); +//! +//! // Name will be suffixed by the current time it was generated +//! let timestamped_name = naming::generate_timestamped_name("test", "pdf"); +//! +//! // Random name is a UUIDv4 string suffixed by the extension +//! let random_name = naming::generate_random_name("pdf"); +//! +//! // N-digit name is a number prefixed by X zeros +//! let n_digit_name = naming::generate_n_digit_name(5, 4, "pdf"); +//! } +//! ``` +//! + +use chrono::prelude::*; +use std::path::PathBuf; +use uuid::Uuid; + +/// Generates a `PathBuf` from a given and extension +/// +/// Returns a `PathBuf` of the form `name.ext` +pub fn generate_name(name: &str, ext: &str) -> PathBuf { + PathBuf::from(format!("{}.{}", name, ext)) +} + +/// Generates a `PathBuf` from a name and extention with a default timestamp of "DD_MM_YY_HHMMSS" +/// If `fname` is "", just uses the timestamp and extension +/// +/// Returns `PathBuf` in the form `fname_timestamp.ext` +pub fn generate_timestamped_name(fname: &str, ext: &str) -> PathBuf { + let dt = UTC::now().format("%d_%m_%Y_%Hh%Mm%Ss"); + + if fname.len() == 0 { + return PathBuf::from(format!("{}.{}", dt, ext)); + } + + PathBuf::from(format!("{}_{}.{}", fname, dt, ext)) +} + +/// Generates a random UUIDv4 `PathBuf` +/// +/// Returns `PathBuf` in the form `uuid.ext` +pub fn generate_random_name(ext: &str) -> PathBuf { + let unique = Uuid::new_v4(); + + PathBuf::from(format!("{}.{}", unique.to_string(), ext)) +} + +/// Generates a `PathBuf` from a `number` prefixed by `n_digits` zeros +/// +/// Returns `PathBuf` of the form e.g `0005.ext` +pub fn generate_n_digit_name(number: i32, n_digits: usize, ext: &str) -> PathBuf { + PathBuf::from(format!("{:0fill$}.{}", number, ext, fill = n_digits)) +} + +#[cfg(test)] +mod naming_tests { + use super::*; + use regex::Regex; + use std::path::PathBuf; + + #[test] + fn generates_expected_name() { + assert_eq!(generate_name("test", "pdf"), PathBuf::from("test.pdf")); + assert_eq!( + generate_name("another", "txt"), + PathBuf::from("another.txt") + ); + assert_eq!(generate_name("main", "c"), PathBuf::from("main.c")); + assert_eq!(generate_name("app", "js"), PathBuf::from("app.js")); + assert_eq!( + generate_name("somephotothing", "H4AC"), + PathBuf::from("somephotothing.H4AC") + ); + } + + #[test] + // Don't judge me on regex... + fn generates_timestamped_name_ok() { + let ts_re = Regex::new(r"(.*)_\d{2}_\d{2}_\d{4}_\d{2}h\d{2}m\d{2}s\.(.*)").unwrap(); + let ts_name = generate_timestamped_name("with_filename", "txt"); + eprintln!("Testname: {ts_name:?}"); + + // Pathbuf checks need the full path component + let ts_name = ts_name.to_str().unwrap(); + assert!(ts_name.starts_with("with_filename")); + assert!(ts_re.is_match(ts_name)); + assert!(ts_name.ends_with(".txt")); + + let no_prefix_re = Regex::new(r"\d{2}_\d{2}_\d{4}_\d{2}h\d{2}m\d{2}s\.(.*)").unwrap(); + let no_prefix = generate_timestamped_name("", "pdf"); + + let no_prefix = no_prefix.to_str().unwrap(); + assert!(no_prefix.ends_with("pdf")); + assert!(no_prefix_re.is_match(no_prefix)); + } + + #[test] + fn checks_random_names_are_ok() { + let rn = generate_random_name("json"); + // TODO: Add regex test to match UUIDv4 Pattern + } +} From f0083399ae864ba8d955e28203a507ced49ae594 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Mon, 5 Feb 2024 18:05:57 +0000 Subject: [PATCH 078/105] finished tests for naming, removed unused files --- src/filehelpers.rs | 167 --------------------------------------------- src/naming.rs | 37 +++++++--- 2 files changed, 28 insertions(+), 176 deletions(-) delete mode 100644 src/filehelpers.rs diff --git a/src/filehelpers.rs b/src/filehelpers.rs deleted file mode 100644 index 3a3574d..0000000 --- a/src/filehelpers.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! Functions that help in iterating files and folders -//! -//! # Examples -//! -//! ``` -//! use std::path::PathBuf; -//! use filetools::filehelpers; -//! -//! fn main() -> Result<(), Box> { -//! /// Creating a directory -//! let new_path = PathBuf::from("./test"); -//! let _ = filehelpers::ensure_dir(new_path)?; -//! -//! /// Iterating through all files in a directory -//! let nr_search = PathBuf::from("./test"); -//! let r_search = PathBuf::from("./test"); -//! -//! // Non-recursive search of directroy, just files in search folder -//! let non_recursed_files = filehelpers::list_files(nr_search, false); -//! -//! // Recursive search of directory, gets all files in directory and all sub-directories -//! let recursed_files = filehelpers::list_files(r_search, true); -//! -//! /// Iterating through all folders in a directory -//! let nr_search = PathBuf::from("./test"); -//! let r_search = PathBuf::from("./test"); -//! -//! // Non-recursive search for all folders, just folders in search directory -//! let non_recursive_folders = filehelpers::list_folders(nr_search, false); -//! -//! // Recursive search of all folders, all subfolders in a directory as well -//! let recursive_folders = filehelpers::list_folders(r_search, true); -//! -//! Ok(()) -//! } -//! ``` -//! - -use std::fs; -use std::path::{Path, PathBuf}; - -/// Ensures a directory is created from a `PathBuf` -/// Does nothing if the directory already exists -/// -/// Returns `Ok` if successful, `Err` if not -pub fn ensure_dir(dir_name: PathBuf) -> Result <(), Box> { - let path = Path::new(&dir_name); - if !path.exists() { - fs::create_dir(path)?; - } - - Ok(()) -} - -/// Determines if a `path` if a subdirectory of the given `directory` -/// Creates the absolute paths and checks the `ancestors` of `path` to determine if a subdirectory -/// -/// Note::Not entirely sure this works perfectly fine, use at own risk -/// -/// Returns `Ok(true)` if `path` is a subdirectory, `Ok(false)` if not, `Err` if error occured -pub fn is_subdir(path: PathBuf, directory: PathBuf) -> Result> { - // Get absolute paths - let directory = fs::canonicalize(Path::new(&directory))?; - let path = fs::canonicalize(Path::new(&path))?; - - let mut is_subdir = Ok(false); - - // Iterate through all ancestors of the path - for ancestor in path.ancestors() { - // Found directory, current path is a subdirectory - if ancestor == directory { - is_subdir = Ok(true); - break; - } - } - - is_subdir -} - -/// Determines if a given `PathBuf` contains a search string -/// -/// Returns `true` if search string present, else `false` -pub fn path_contains(path: PathBuf, search_str: &str) -> bool { - // Path successfully converted to str - if let Some(p) = path.to_str() { - // Contains string, return true - if p.contains(search_str) { - return true - } - } - - // Search string not found - false -} - -/// Lists all files in a given `path` -/// If `recursive` is set, iterates through all subfolders recursively to find all files -/// If `recursive` not set, just finds all files in the current directory -/// -/// Return `Vec` of all files in a directory and subdirectories -pub fn list_files(path: PathBuf, recursive: bool) -> Result, Box> { - let mut found_files = Vec::new(); - let search_path = Path::new(&path); - - // Iterate through all entries in the directory - for entry in fs::read_dir(search_path)? { - // Get File metadata - let entry = entry?; - let path = entry.path(); - let metadata = fs::metadata(&path)?; - - // Entry is a file, add to array - if metadata.is_file() { - found_files.push(path); - } else if metadata.is_dir() && recursive { - // Found a directory and recursively looking - let subfiles = list_files(path, recursive)?; - - // Add all found subfiles to array - for file in subfiles.iter() { - found_files.push(file.to_path_buf()); - } - } else { - continue; - } - } - - Ok(found_files) -} - -/// Lists all folders in a given `path` -/// If `recursive` is set, iterates through all subfolders recursively to find all folders -/// If `recursive` not set, just finds all files in the current directory -/// Mirrors the functionality of `filehelpers::list_files()` -/// -/// Return `Vec` of all folders in a directory and subdirectories -pub fn list_folders(path: PathBuf, recursive: bool) -> Result, Box> { - let mut found_folders = Vec::new(); - let search_path = Path::new(&path); - - // Iterate through all entries in the directory - for entry in fs::read_dir(search_path)? { - // Get File metadata - let entry = entry?; - let path = entry.path(); - let metadata = fs::metadata(&path)?; - - // Entry is a directory, add to array - if metadata.is_dir() { - found_folders.push(path); - - // Recursively looking - if recursive { - // Search recursively - let f_path = entry.path(); - let subfolders = list_folders(f_path, recursive)?; - - // Add all subfolders to array - for subfolder in subfolders.iter() { - found_folders.push(subfolder.to_path_buf()); - } - } - } - } - - Ok(found_folders) -} \ No newline at end of file diff --git a/src/naming.rs b/src/naming.rs index abf7f38..c2ab93d 100644 --- a/src/naming.rs +++ b/src/naming.rs @@ -25,11 +25,22 @@ use chrono::prelude::*; use std::path::PathBuf; use uuid::Uuid; +/// Helper for makeing extensions +/// +/// Literally just preprends a . +fn make_extension(ext: impl AsRef) -> String { + if ext.as_ref().is_empty() { + return String::new(); + } + + format!(".{}", ext.as_ref()) +} + /// Generates a `PathBuf` from a given and extension /// /// Returns a `PathBuf` of the form `name.ext` pub fn generate_name(name: &str, ext: &str) -> PathBuf { - PathBuf::from(format!("{}.{}", name, ext)) + PathBuf::from(format!("{}{}", name, make_extension(ext))) } /// Generates a `PathBuf` from a name and extention with a default timestamp of "DD_MM_YY_HHMMSS" @@ -40,10 +51,10 @@ pub fn generate_timestamped_name(fname: &str, ext: &str) -> PathBuf { let dt = UTC::now().format("%d_%m_%Y_%Hh%Mm%Ss"); if fname.len() == 0 { - return PathBuf::from(format!("{}.{}", dt, ext)); + return PathBuf::from(format!("{}{}", dt, make_extension(ext))); } - PathBuf::from(format!("{}_{}.{}", fname, dt, ext)) + PathBuf::from(format!("{}_{}{}", fname, dt, make_extension(ext))) } /// Generates a random UUIDv4 `PathBuf` @@ -52,14 +63,19 @@ pub fn generate_timestamped_name(fname: &str, ext: &str) -> PathBuf { pub fn generate_random_name(ext: &str) -> PathBuf { let unique = Uuid::new_v4(); - PathBuf::from(format!("{}.{}", unique.to_string(), ext)) + PathBuf::from(format!("{}{}", unique.to_string(), make_extension(ext))) } /// Generates a `PathBuf` from a `number` prefixed by `n_digits` zeros /// /// Returns `PathBuf` of the form e.g `0005.ext` pub fn generate_n_digit_name(number: i32, n_digits: usize, ext: &str) -> PathBuf { - PathBuf::from(format!("{:0fill$}.{}", number, ext, fill = n_digits)) + PathBuf::from(format!( + "{:0fill$}{}", + number, + make_extension(ext), + fill = n_digits + )) } #[cfg(test)] @@ -86,9 +102,8 @@ mod naming_tests { #[test] // Don't judge me on regex... fn generates_timestamped_name_ok() { - let ts_re = Regex::new(r"(.*)_\d{2}_\d{2}_\d{4}_\d{2}h\d{2}m\d{2}s\.(.*)").unwrap(); + let ts_re = Regex::new(r"(.*)_\d{2}_\d{2}_\d{4}_\d{2}h\d{2}m\d{2}s").unwrap(); let ts_name = generate_timestamped_name("with_filename", "txt"); - eprintln!("Testname: {ts_name:?}"); // Pathbuf checks need the full path component let ts_name = ts_name.to_str().unwrap(); @@ -96,7 +111,7 @@ mod naming_tests { assert!(ts_re.is_match(ts_name)); assert!(ts_name.ends_with(".txt")); - let no_prefix_re = Regex::new(r"\d{2}_\d{2}_\d{4}_\d{2}h\d{2}m\d{2}s\.(.*)").unwrap(); + let no_prefix_re = Regex::new(r"\d{2}_\d{2}_\d{4}_\d{2}h\d{2}m\d{2}s").unwrap(); let no_prefix = generate_timestamped_name("", "pdf"); let no_prefix = no_prefix.to_str().unwrap(); @@ -106,7 +121,11 @@ mod naming_tests { #[test] fn checks_random_names_are_ok() { + let uuid_re = + Regex::new(r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}").unwrap(); let rn = generate_random_name("json"); - // TODO: Add regex test to match UUIDv4 Pattern + let rn_name = rn.to_str().unwrap(); + assert!(uuid_re.is_match(rn_name)); + assert!(rn_name.ends_with(".json")); } } From 7105607e3084309c96ab2e48fdce84039b22d0dd Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Tue, 6 Feb 2024 18:54:51 +0000 Subject: [PATCH 079/105] added sync versions of ft functions --- src/lib.rs | 18 ++++++------ src/sync.rs | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 src/sync.rs diff --git a/src/lib.rs b/src/lib.rs index 980d82c..52562a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,9 @@ use async_recursion::async_recursion; use std::path::{Component, Path, PathBuf}; use tokio::fs; +pub mod file; pub mod naming; +pub mod sync; // What do we want? // @@ -20,7 +22,7 @@ pub mod naming; // File Task Manager - That operation scheduler thing #[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum FtIterItemState { +pub(crate) enum FtIterItemState { FileNoRec, FileRec, DirNoRec, @@ -37,19 +39,19 @@ pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { Ok(()) } -pub async fn is_subdir(path: impl AsRef, dir: impl AsRef) -> Result { +pub fn is_subdir(path: impl AsRef, dir: impl AsRef) -> bool { for component in path.as_ref().components() { match component { Component::Normal(p) => { if p == dir.as_ref().as_os_str() { - return Ok(true); + return true; } } _ => {} } } - Ok(false) + false } pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe */) -> bool { @@ -250,15 +252,11 @@ mod tests { let nested = root .nest_folders(vec!["this", "is", "a", "nested", "tmp", "dir"]) .await?; - let mut result = is_subdir(&nested.path, "nested") - .await - .context("is_subdir test")?; + let mut result = is_subdir(&nested.path, "nested"); assert!(result); - result = is_subdir(&nested.path, "not_valid") - .await - .context("is_subdir test")?; + result = is_subdir(&nested.path, "not_valid"); assert!(!result); Ok(()) diff --git a/src/sync.rs b/src/sync.rs new file mode 100644 index 0000000..2f8986e --- /dev/null +++ b/src/sync.rs @@ -0,0 +1,83 @@ +use crate::FtIterItemState; +use anyhow::{Context, Result}; +use std::fs; +use std::path::{Path, PathBuf}; + +pub fn ensure_directory(dir: impl AsRef) -> Result<()> { + if !dir.as_ref().exists() { + fs::create_dir_all(dir).context("unable to create directory")?; + } + + Ok(()) +} + +pub fn list_files>(path: P) -> Result>> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems(path, FtIterItemState::FileNoRec) +} + +pub fn list_folders>(path: P) -> Result>> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + iteritems(path, FtIterItemState::DirNoRec) +} + +pub fn list_files_recursive>(path: P) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems(path, FtIterItemState::FileRec) +} + +pub fn list_folders_recursive + Send>(path: P) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + iteritems(path, FtIterItemState::DirRec) +} + +fn iteritems>(path: P, iterstate: FtIterItemState) -> Result> { + let mut items = vec![]; + + let mut entries = fs::read_dir(path.as_ref()).context("list items inner call")?; + + while let Some(Ok(entry)) = entries.next() { + let e_path = entry.path(); + match iterstate { + FtIterItemState::FileNoRec => { + if e_path.is_file() { + items.push(e_path); + } + } + FtIterItemState::FileRec => { + if e_path.is_file() { + items.push(e_path) + } else if e_path.is_dir() { + items.extend(iteritems(e_path, iterstate)?); + } + } + FtIterItemState::DirNoRec => { + if e_path.is_dir() { + items.push(e_path); + } + } + FtIterItemState::DirRec => { + if e_path.is_dir() { + items.push(e_path.clone()); + items.extend(iteritems(e_path, iterstate)?); + } + } + } + } + + Ok(items) +} + +// Do I write tests for these as they are just the sync version of the Ft functions +// which already pass...? +// TODO: Maybe tests? From 3086b4276a06d091b9f660f9104d1a7baa2f2f46 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Tue, 6 Feb 2024 20:49:08 +0000 Subject: [PATCH 080/105] added function to create bulk numeric directories --- src/lib.rs | 58 ++++++++++++++++++++++++++++++++++++++++----------- src/naming.rs | 6 +++--- src/sync.rs | 16 +++++++++++++- 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 52562a0..b20860e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,16 +29,6 @@ pub(crate) enum FtIterItemState { DirRec, } -pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { - if !dir.as_ref().exists() { - fs::create_dir_all(dir) - .await - .context("unable to create directory")?; - } - - Ok(()) -} - pub fn is_subdir(path: impl AsRef, dir: impl AsRef) -> bool { for component in path.as_ref().components() { match component { @@ -64,7 +54,35 @@ pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe false } -pub async fn list_files + Send>(path: P) -> Result>> { +pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { + if !dir.as_ref().exists() { + fs::create_dir_all(dir) + .await + .context("unable to create directory")?; + } + + Ok(()) +} + +pub async fn create_numeric_directories( + path: impl AsRef, + start: usize, + end: usize, + fill: usize, +) -> Result<()> { + for i in start..end { + let name = path + .as_ref() + .join(naming::generate_n_digit_name(i, fill, "")); + ensure_directory(name) + .await + .context("creating numeric directories")?; + } + + Ok(()) +} + +pub async fn list_files + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), @@ -74,7 +92,7 @@ pub async fn list_files + Send>(path: P) -> Result + Send>(path: P) -> Result>> { +pub async fn list_folders + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirNoRec).await } @@ -360,4 +378,20 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn numeric_directories() -> Result<()> { + let tmp = TempPath::new("numeric_directories").await?; + create_numeric_directories(&tmp.path, 0, 100, 4).await?; + let mut folders = list_folders(&tmp.path).await?; + folders.sort(); + assert_eq!(folders.len(), 100); + + for (i, folder) in folders.into_iter().enumerate() { + let test = &tmp.path.join(format!("{:0fill$}", i, fill = 4)); + assert_eq!(&folder, test); + } + + Ok(()) + } } diff --git a/src/naming.rs b/src/naming.rs index c2ab93d..5520381 100644 --- a/src/naming.rs +++ b/src/naming.rs @@ -50,7 +50,7 @@ pub fn generate_name(name: &str, ext: &str) -> PathBuf { pub fn generate_timestamped_name(fname: &str, ext: &str) -> PathBuf { let dt = UTC::now().format("%d_%m_%Y_%Hh%Mm%Ss"); - if fname.len() == 0 { + if fname.is_empty() { return PathBuf::from(format!("{}{}", dt, make_extension(ext))); } @@ -69,12 +69,12 @@ pub fn generate_random_name(ext: &str) -> PathBuf { /// Generates a `PathBuf` from a `number` prefixed by `n_digits` zeros /// /// Returns `PathBuf` of the form e.g `0005.ext` -pub fn generate_n_digit_name(number: i32, n_digits: usize, ext: &str) -> PathBuf { +pub fn generate_n_digit_name(number: usize, fill: usize, ext: &str) -> PathBuf { PathBuf::from(format!( "{:0fill$}{}", number, make_extension(ext), - fill = n_digits + fill = fill )) } diff --git a/src/sync.rs b/src/sync.rs index 2f8986e..83a7ece 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,4 +1,4 @@ -use crate::FtIterItemState; +use crate::{naming::generate_n_digit_name, FtIterItemState}; use anyhow::{Context, Result}; use std::fs; use std::path::{Path, PathBuf}; @@ -21,6 +21,20 @@ pub fn list_files>(path: P) -> Result>> { iteritems(path, FtIterItemState::FileNoRec) } +pub fn create_numeric_directories( + path: impl AsRef, + start: usize, + end: usize, + fill: usize, +) -> Result<()> { + for i in start..end { + let name = path.as_ref().join(generate_n_digit_name(i, fill, "")); + ensure_directory(name).context("creating numeric directories")?; + } + + Ok(()) +} + pub fn list_folders>(path: P) -> Result>> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirNoRec) From 71a9aa59eac924c0cb5610bd720f4ada6b67551c Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Tue, 6 Feb 2024 21:51:55 +0000 Subject: [PATCH 081/105] working on docs --- Cargo.toml | 2 +- src/lib.rs | 196 +++++++++++++++++++++++++++++++++++++++++++++++++--- src/sync.rs | 20 +++++- 3 files changed, 204 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cd8f7ce..683b784 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,5 @@ tokio = { version = "1.35.1", features = ["fs"] } uuid = { version = "0.8.1", features = ["v4"]} [dev-dependencies] -tokio = { version = "1.35.1", features = ["macros", "rt"] } +tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread"] } regex = "1.10.3" diff --git a/src/lib.rs b/src/lib.rs index b20860e..48f107a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,6 @@ +//! Crate to help with simple file / folder operations +//! +//! TODO: More Docs! use anyhow::{Context, Result}; use async_recursion::async_recursion; use std::path::{Component, Path, PathBuf}; @@ -10,8 +13,6 @@ pub mod sync; // What do we want? // // MUST: -// Naming helpers -// Sync functionality of the current operations // Docs etc. // // NICE: @@ -21,14 +22,37 @@ pub mod sync; // MILES OFF: // File Task Manager - That operation scheduler thing +/// Determines the type of iteration performed by the `list_directories` and `list_files` functions +/// If the NoRec variation is used, only the current directory is considered +/// If the Rec variation is used, then ALL subdirectores are traversed #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum FtIterItemState { + /// Iterate files with no recursion FileNoRec, + + /// Iterate files with recursion FileRec, + + /// Iterate directories with no recursion DirNoRec, + + /// Iterate directories with recursion DirRec, } +/// Checks if a given pattern is considered a subdirectory of the given path +/// +/// # Example +/// +/// ```rust +/// use filetools::is_subdir; +/// +/// let path = "directory/to/check/for/sub/directory"; +/// let check = "for"; +/// +/// // As "for" is a subdirectory in this path, this returns true +/// let result = is_subdir(path, check); +/// ``` pub fn is_subdir(path: impl AsRef, dir: impl AsRef) -> bool { for component in path.as_ref().components() { match component { @@ -44,6 +68,21 @@ pub fn is_subdir(path: impl AsRef, dir: impl AsRef) -> bool { false } +/// Determines if a path contains a given pattern +/// +/// Converts both the path and the pattern to a string and performs simple matching +/// +/// # Example +/// +/// ```rust +/// use filetools::path_contains; +/// +/// let path = "This/is/a/path/with/a/file.txt"; +/// let pattern = "file.txt"; +/// +/// // The path contains the pattern file.txt so this returns true +/// let result = path_contains(path, pattern); +/// ``` pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe */) -> bool { if let Some(p) = path.as_ref().to_str() { if let Some(pat) = pattern.as_ref().to_str() { @@ -54,6 +93,23 @@ pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe false } +/// Creates a directory at the given path. +/// +/// If the directory already exists, nothing is done +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::ensure_directory; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let target_path = "directory/to/create"; +/// ensure_directory(target_path).await?; +/// +/// Ok(()) +/// } +/// ``` pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { if !dir.as_ref().exists() { fs::create_dir_all(dir) @@ -64,6 +120,28 @@ pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { Ok(()) } +/// Creates a range of numeric folders in the given path starting from `start` +/// up to `end` (non-inclusive). +/// +/// Directories can be padded with X zeros using the `fill` parameter. +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::create_numeric_directories; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let root = "some/root/path"; +/// +/// // This will create the following directories: +/// // "some/root/path/0000" +/// // ... +/// // "some/root/path/0099" +/// create_numeric_directories(root, 0, 100, 4).await?; +/// Ok(()) +/// } +/// ``` pub async fn create_numeric_directories( path: impl AsRef, start: usize, @@ -82,6 +160,29 @@ pub async fn create_numeric_directories( Ok(()) } +/// Lists all files in the given directory (not including subdirectories). +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The path given is a file and not a directory +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::list_files; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let target_folder = "folder/containing/files"; +/// +/// // Will return a Vec containing all files in the folder +/// let files = list_files(target_folder).await?; +/// Ok(()) +/// } +/// ``` pub async fn list_files + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( @@ -92,11 +193,57 @@ pub async fn list_files + Send>(path: P) -> Result> iteritems(path, FtIterItemState::FileNoRec).await } -pub async fn list_folders + Send>(path: P) -> Result> { +/// Lists all directories in the given directory (not including subdirectories). +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::list_directories; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let target_folder = "directory/containing/other/directories"; +/// +/// // Will return a Vec containing all directories in the folder +/// let directories = list_directories(target_folder).await?; +/// Ok(()) +/// } +/// ``` +pub async fn list_directories + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirNoRec).await } +/// Lists all files in a directory including ALL subdirectories +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The given path is a file and not a directory +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::list_files_recursive; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let target_folder = "directory/containing/nested/files"; +/// +/// // This will return a Vec of ALL files contained within the directory +/// // (including in all subdirectories) +/// let files = list_files_recursive(target_folder).await?; +/// Ok(()) +/// } +/// ``` #[async_recursion] pub async fn list_files_recursive + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); @@ -108,12 +255,37 @@ pub async fn list_files_recursive + Send>(path: P) -> Result anyhow::Result<()> { +/// let target_folder = "directory/containing/nested/files"; +/// +/// // This will return a Vec of ALL directories contained within the directory +/// // (including in all subdirectories) +/// let directories = list_directories_recursive(target_folder).await?; +/// Ok(()) +/// } +/// ``` #[async_recursion] -pub async fn list_folders_recursive + Send>(path: P) -> Result> { +pub async fn list_directories_recursive + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirRec).await } +/// Helper function to iterate through a directory to find all Files / Directories +/// depending on the `FilterState` passed. #[async_recursion] async fn iteritems + Send>( path: P, @@ -347,21 +519,21 @@ mod tests { } #[tokio::test] - async fn check_list_folders_works() -> Result<()> { + async fn check_list_directories_works() -> Result<()> { let root = TempPath::new("lfolder_test").await?; root.multi_folder(vec!["folder1", "folder2", "folder3", "folder4"]) .await?; - let res = list_folders(root.path.clone()).await?; + let res = list_directories(root.path.clone()).await?; assert_eq!(res.len(), 4); - assert!(list_folders("non-existant_path").await.is_err()); + assert!(list_directories("non-existant_path").await.is_err()); Ok(()) } #[tokio::test] - async fn check_list_folders_recursive_works() -> Result<()> { + async fn check_list_directories_recursive_works() -> Result<()> { let root = TempPath::new("lfolderrec_test").await?; root.multi_folder(vec!["folder1", "folder2"]).await?; @@ -371,10 +543,12 @@ mod tests { let s2 = TempPath::new(f1.join("sub2")).await?; s2.multi_folder(vec!["deep1", "deep2"]).await?; - let res = list_folders_recursive(root.path.clone()).await?; + let res = list_directories_recursive(root.path.clone()).await?; assert_eq!(res.len(), 7); - assert!(list_folders_recursive("not-a-valId_pathd").await.is_err()); + assert!(list_directories_recursive("not-a-valId_pathd") + .await + .is_err()); Ok(()) } @@ -383,7 +557,7 @@ mod tests { async fn numeric_directories() -> Result<()> { let tmp = TempPath::new("numeric_directories").await?; create_numeric_directories(&tmp.path, 0, 100, 4).await?; - let mut folders = list_folders(&tmp.path).await?; + let mut folders = list_directories(&tmp.path).await?; folders.sort(); assert_eq!(folders.len(), 100); diff --git a/src/sync.rs b/src/sync.rs index 83a7ece..73214a1 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,8 +1,24 @@ +//! Sync variations of the main [`crate`] functions use crate::{naming::generate_n_digit_name, FtIterItemState}; use anyhow::{Context, Result}; use std::fs; use std::path::{Path, PathBuf}; +/// Creates a directory at the given path. +/// +/// If the directory already exists, nothing is done +/// +/// This is the sync version of [`crate::ensure_directory`] +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::sync::ensure_directory; +/// +/// let target_path = "directory/to/create"; +/// ensure_directory(target_path).expect("unable to create directory"); +/// +/// ``` pub fn ensure_directory(dir: impl AsRef) -> Result<()> { if !dir.as_ref().exists() { fs::create_dir_all(dir).context("unable to create directory")?; @@ -35,7 +51,7 @@ pub fn create_numeric_directories( Ok(()) } -pub fn list_folders>(path: P) -> Result>> { +pub fn list_directories>(path: P) -> Result>> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirNoRec) } @@ -50,7 +66,7 @@ pub fn list_files_recursive>(path: P) -> Result> { iteritems(path, FtIterItemState::FileRec) } -pub fn list_folders_recursive + Send>(path: P) -> Result> { +pub fn list_directories_recursive + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirRec) } From 25d931c043caf91addfc1278f2b8b0fe0763edd8 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Thu, 8 Feb 2024 12:17:50 +0000 Subject: [PATCH 082/105] docs: updated docs and better naming --- src/lib.rs | 6 +++ src/naming.rs | 59 ++++++++++++++++++++---- src/sync.rs | 124 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 169 insertions(+), 20 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 48f107a..4f375f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -199,6 +199,7 @@ pub async fn list_files + Send>(path: P) -> Result> /// /// This function will return an error in the following situations: /// +/// * The given path is a file and not a directory /// * The given path does not exist /// /// # Example @@ -217,6 +218,11 @@ pub async fn list_files + Send>(path: P) -> Result> /// ``` pub async fn list_directories + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + iteritems(path, FtIterItemState::DirNoRec).await } diff --git a/src/naming.rs b/src/naming.rs index 5520381..673ddbc 100644 --- a/src/naming.rs +++ b/src/naming.rs @@ -7,15 +7,18 @@ //! use filetools::naming; //! //! fn main() { +//! // Generates the name `test.pdf` //! let custom_name = naming::generate_name("test", "pdf"); //! //! // Name will be suffixed by the current time it was generated +//! // E.g. `test_[Timestamp].pdf` //! let timestamped_name = naming::generate_timestamped_name("test", "pdf"); //! //! // Random name is a UUIDv4 string suffixed by the extension -//! let random_name = naming::generate_random_name("pdf"); +//! // E.g. `00762527-012a-43c1-a673-cad9bc5eef64.pdf` +//! let random_name = naming::generate_uuid4_name("pdf"); //! -//! // N-digit name is a number prefixed by X zeros +//! // N-digit name is a number prefixed by X zeros (e.g. 0005.pdf) //! let n_digit_name = naming::generate_n_digit_name(5, 4, "pdf"); //! } //! ``` @@ -38,15 +41,32 @@ fn make_extension(ext: impl AsRef) -> String { /// Generates a `PathBuf` from a given and extension /// -/// Returns a `PathBuf` of the form `name.ext` +/// # Example +/// +/// ```rust +/// use filetools::naming::generate_name; +/// +/// // Will generate the name `test.json` +/// let name = generate_name("test", "json"); +/// ``` pub fn generate_name(name: &str, ext: &str) -> PathBuf { PathBuf::from(format!("{}{}", name, make_extension(ext))) } /// Generates a `PathBuf` from a name and extention with a default timestamp of "DD_MM_YY_HHMMSS" -/// If `fname` is "", just uses the timestamp and extension +/// If `fname` is empty, just uses the timestamp and extension +/// +/// # Example /// -/// Returns `PathBuf` in the form `fname_timestamp.ext` +/// ```rust +/// use filetools::naming::generate_timestamped_name; +/// +/// // Will generate the name `some_file_[Timestamp].pdf` +/// let ts_with_filename = generate_timestamped_name("some_file", ".pdf"); +/// +/// // Will generate the name `[Timestamp].txt` +/// let ts_no_filename = generate_timestamped_name("", ".txt"); +/// ``` pub fn generate_timestamped_name(fname: &str, ext: &str) -> PathBuf { let dt = UTC::now().format("%d_%m_%Y_%Hh%Mm%Ss"); @@ -59,16 +79,35 @@ pub fn generate_timestamped_name(fname: &str, ext: &str) -> PathBuf { /// Generates a random UUIDv4 `PathBuf` /// -/// Returns `PathBuf` in the form `uuid.ext` -pub fn generate_random_name(ext: &str) -> PathBuf { +/// # Example +/// +/// ```rust +/// use filetools::naming::generate_uuid4_name; +/// +/// // Will generate a UUIDv4 name (e.g. `b1faa2c3-d25c-43bb-b578-9f259d7aabaf.log`) +/// let name = generate_uuid4_name("log"); +/// ``` +pub fn generate_uuid4_name(ext: &str) -> PathBuf { let unique = Uuid::new_v4(); PathBuf::from(format!("{}{}", unique.to_string(), make_extension(ext))) } -/// Generates a `PathBuf` from a `number` prefixed by `n_digits` zeros +/// Generates a `PathBuf` from a `number` prefixed by `n_digits` zeros. +/// +/// If `ext` is empty, will just return the filled number. +/// +/// # Example +/// +/// ```rust +/// use filetools::naming::generate_n_digit_name; +/// +/// // Will generate the name `0005.json` +/// let name = generate_n_digit_name(5, 4, "json"); /// -/// Returns `PathBuf` of the form e.g `0005.ext` +/// // Will generate the name `000128.log` +/// let another_name = generate_n_digit_name(128, 6, "log"); +/// ``` pub fn generate_n_digit_name(number: usize, fill: usize, ext: &str) -> PathBuf { PathBuf::from(format!( "{:0fill$}{}", @@ -123,7 +162,7 @@ mod naming_tests { fn checks_random_names_are_ok() { let uuid_re = Regex::new(r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}").unwrap(); - let rn = generate_random_name("json"); + let rn = generate_uuid4_name("json"); let rn_name = rn.to_str().unwrap(); assert!(uuid_re.is_match(rn_name)); assert!(rn_name.ends_with(".json")); diff --git a/src/sync.rs b/src/sync.rs index 73214a1..d462bc5 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -27,16 +27,26 @@ pub fn ensure_directory(dir: impl AsRef) -> Result<()> { Ok(()) } -pub fn list_files>(path: P) -> Result>> { - anyhow::ensure!(path.as_ref().exists(), "path does not exist"); - anyhow::ensure!( - path.as_ref().is_dir(), - "path should be a directory, not a file" - ); - - iteritems(path, FtIterItemState::FileNoRec) -} - +/// Creates a range of numeric folders in the given path starting from `start` +/// up to `end` (non-inclusive). +/// +/// Directories can be padded with X zeros using the `fill` parameter. +/// +/// This is the sync version of [`crate::create_numeric_directories`] +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::sync::create_numeric_directories; +/// +/// let root = "some/root/path"; +/// +/// // This will create the following directories: +/// // "some/root/path/0000" +/// // ... +/// // "some/root/path/0099" +/// create_numeric_directories(root, 0, 100, 4).expect("unable to create numeric directories"); +/// ``` pub fn create_numeric_directories( path: impl AsRef, start: usize, @@ -51,11 +61,82 @@ pub fn create_numeric_directories( Ok(()) } +/// Lists all files in the given directory (not including subdirectories). +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The path given is a file and not a directory +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::sync::list_files; +/// +/// let target_dir = "some/dir/containing/files"; +/// +/// // Will return a Vec containing paths to all files in the directory +/// let files = list_files(target_dir).expect("unable to list files"); +/// ``` +pub fn list_files>(path: P) -> Result>> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems(path, FtIterItemState::FileNoRec) +} + +/// Lists all directories in the given directory (not including subdirectories). +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The path given is a file and not a directory +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::sync::list_directories; +/// +/// let target_dir = "some/dir/containing/files"; +/// +/// // Will return a Vec containing paths to all directories in the directory +/// let dirs = list_directories(target_dir).expect("unable to list directories"); +/// ``` pub fn list_directories>(path: P) -> Result>> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); iteritems(path, FtIterItemState::DirNoRec) } +/// Lists all files in a directory including ALL subdirectories +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The given path is a file and not a directory +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::sync::list_files_recursive; +/// +/// let target_dir = "some/dir/containing/nested/files"; +/// +/// // Will return a Vec containing all files in the directory (including all subdirectories) +/// let files = list_files_recursive(target_dir).expect("unable to list files recursively"); +/// ``` pub fn list_files_recursive>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( @@ -66,8 +147,31 @@ pub fn list_files_recursive>(path: P) -> Result> { iteritems(path, FtIterItemState::FileRec) } +/// Lists all directories in a directory including ALL subdirectories +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The given path is a file and not a directory +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::sync::list_directories_recursive; +/// +/// let target_dir = "some/dir/containing/nested/files"; +/// +/// // Will return a Vec containing all directories in the directory (including all subdirectories) +/// let dirs = list_directories_recursive(target_dir).expect("unable to list directories recursively"); +/// ``` pub fn list_directories_recursive + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); iteritems(path, FtIterItemState::DirRec) } From 38aecbc338ca407a488021929e2043fc454472e1 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Thu, 8 Feb 2024 17:18:10 +0000 Subject: [PATCH 083/105] added function to create multiple directories --- src/lib.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++- src/sync.rs | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 4f375f8..ea52391 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,11 @@ -//! Crate to help with simple file / folder operations +//! Crate to help with simple file / folder operations. +//! +//! Provides helper functions to: +//! +//! * Create directories +//! * Check filepaths contain a pattern +//! * List files / directories both iteratively and recursively +//! * Generate names for files / directories //! //! TODO: More Docs! use anyhow::{Context, Result}; @@ -120,6 +127,39 @@ pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { Ok(()) } +/// Creates multiple directories inside the target path. +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::create_multiple_directories; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let root = "dir/to/populate"; +/// let to_create = ["dir1", "dir2", "dir3"]; +/// +/// // Will create: +/// // `dir/to/populate/dir1` +/// // `dir/to/populate/dir2` +/// // `dir/to/populate/dir3` +/// create_multiple_directories(root, &to_create); +/// +/// Ok(()) +/// } +/// ``` +pub async fn create_multiple_directories( + path: impl AsRef, + directories: &[impl AsRef], +) -> Result<()> { + for dir in directories { + let target = path.as_ref().join(dir); + ensure_directory(target).await?; + } + + Ok(()) +} + /// Creates a range of numeric folders in the given path starting from `start` /// up to `end` (non-inclusive). /// @@ -574,4 +614,21 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn multiple_directory_creation() -> Result<()> { + let tmp = TempPath::new("create_multiple_dirs").await?; + let dirs = ["config", "src", "tests"]; + + create_multiple_directories(&tmp.path, &dirs).await?; + let folders = list_directories(&tmp.path).await?; + assert_eq!(folders.len(), 3); + + for check in dirs { + let target = tmp.path.join(check); + assert!(folders.contains(&target)); + } + + Ok(()) + } } diff --git a/src/sync.rs b/src/sync.rs index d462bc5..f80eb8c 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -61,8 +61,40 @@ pub fn create_numeric_directories( Ok(()) } +/// Creates multiple directories inside the target path. +/// +/// This is the sync version of [`crate::create_multiple_directories`] +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::sync::create_multiple_directories; +/// +/// let root = "dir/to/populate"; +/// let to_create = ["dir1", "dir2", "dir3"]; +/// +/// // Will create: +/// // `dir/to/populate/dir1` +/// // `dir/to/populate/dir2` +/// // `dir/to/populate/dir3` +/// create_multiple_directories(root, &to_create).expect("unable to create multiple directories"); +/// ``` +pub fn create_multiple_directories( + path: impl AsRef, + directories: &[impl AsRef], +) -> Result<()> { + for dir in directories { + let target = path.as_ref().join(dir); + ensure_directory(target)?; + } + + Ok(()) +} + /// Lists all files in the given directory (not including subdirectories). /// +/// This is the sync version of [`crate::list_files`] +/// /// # Errors /// /// This function will return an error in the following situations: @@ -92,6 +124,8 @@ pub fn list_files>(path: P) -> Result>> { /// Lists all directories in the given directory (not including subdirectories). /// +/// This is the sync version of [`crate::list_directories`] +/// /// # Errors /// /// This function will return an error in the following situations: @@ -120,6 +154,8 @@ pub fn list_directories>(path: P) -> Result> /// Lists all files in a directory including ALL subdirectories /// +/// This is the sync version of [`crate::list_files_recursive`] +/// /// # Errors /// /// This function will return an error in the following situations: @@ -149,6 +185,8 @@ pub fn list_files_recursive>(path: P) -> Result> { /// Lists all directories in a directory including ALL subdirectories /// +/// This is the sync version of [`crate::list_directories_recursive`] +/// /// # Errors /// /// This function will return an error in the following situations: From 55dfc9470f1f12da007ae3d3cec67546f0ef0cf4 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Thu, 8 Feb 2024 19:04:49 +0000 Subject: [PATCH 084/105] changed list_x_recursive to list_nested_x --- src/lib.rs | 37 +++++++++++-------------------------- src/sync.rs | 16 ++++++++-------- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ea52391..6c5797a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,22 +13,9 @@ use async_recursion::async_recursion; use std::path::{Component, Path, PathBuf}; use tokio::fs; -pub mod file; pub mod naming; pub mod sync; -// What do we want? -// -// MUST: -// Docs etc. -// -// NICE: -// Some kind of filtering for searching? -// A higher level File type to perform some ops quicker? -// -// MILES OFF: -// File Task Manager - That operation scheduler thing - /// Determines the type of iteration performed by the `list_directories` and `list_files` functions /// If the NoRec variation is used, only the current directory is considered /// If the Rec variation is used, then ALL subdirectores are traversed @@ -278,7 +265,7 @@ pub async fn list_directories + Send>(path: P) -> Result anyhow::Result<()> { @@ -286,12 +273,12 @@ pub async fn list_directories + Send>(path: P) -> Result + Send>(path: P) -> Result> { +pub async fn list_nested_files + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), @@ -312,7 +299,7 @@ pub async fn list_files_recursive + Send>(path: P) -> Result anyhow::Result<()> { @@ -320,12 +307,12 @@ pub async fn list_files_recursive + Send>(path: P) -> Result + Send>(path: P) -> Result> { +pub async fn list_nested_directories + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirRec).await } @@ -543,7 +530,7 @@ mod tests { } #[tokio::test] - async fn check_list_files_recursive_works() -> Result<()> { + async fn check_list_nested_files_works() -> Result<()> { let root = TempPath::new("lfr_test").await?; let ffolder = root.new_folder("ffolder").await?; let sfolder = root.new_folder("sfolder").await?; @@ -554,7 +541,7 @@ mod tests { sfolder.multi_file(vec!["second.txt", "third.php"]).await?; tfolder.new_file("fourth.cpp").await?; - let res = list_files_recursive(&root.path).await?; + let res = list_nested_files(&root.path).await?; assert_eq!(res.len(), 5); assert!(list_files("IDoNotExistAsADirectoryOrShouldntAtLeAst") @@ -579,7 +566,7 @@ mod tests { } #[tokio::test] - async fn check_list_directories_recursive_works() -> Result<()> { + async fn check_list_nested_directories_works() -> Result<()> { let root = TempPath::new("lfolderrec_test").await?; root.multi_folder(vec!["folder1", "folder2"]).await?; @@ -589,12 +576,10 @@ mod tests { let s2 = TempPath::new(f1.join("sub2")).await?; s2.multi_folder(vec!["deep1", "deep2"]).await?; - let res = list_directories_recursive(root.path.clone()).await?; + let res = list_nested_directories(root.path.clone()).await?; assert_eq!(res.len(), 7); - assert!(list_directories_recursive("not-a-valId_pathd") - .await - .is_err()); + assert!(list_nested_directories("not-a-valId_pathd").await.is_err()); Ok(()) } diff --git a/src/sync.rs b/src/sync.rs index f80eb8c..1d6f3d1 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -154,7 +154,7 @@ pub fn list_directories>(path: P) -> Result> /// Lists all files in a directory including ALL subdirectories /// -/// This is the sync version of [`crate::list_files_recursive`] +/// This is the sync version of [`crate::list_nested_files`] /// /// # Errors /// @@ -166,14 +166,14 @@ pub fn list_directories>(path: P) -> Result> /// # Example /// /// ```rust,no_run -/// use filetools::sync::list_files_recursive; +/// use filetools::sync::list_nested_files; /// /// let target_dir = "some/dir/containing/nested/files"; /// /// // Will return a Vec containing all files in the directory (including all subdirectories) -/// let files = list_files_recursive(target_dir).expect("unable to list files recursively"); +/// let files = list_nested_files(target_dir).expect("unable to list files recursively"); /// ``` -pub fn list_files_recursive>(path: P) -> Result> { +pub fn list_nested_files>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), @@ -185,7 +185,7 @@ pub fn list_files_recursive>(path: P) -> Result> { /// Lists all directories in a directory including ALL subdirectories /// -/// This is the sync version of [`crate::list_directories_recursive`] +/// This is the sync version of [`crate::list_nested_directories`] /// /// # Errors /// @@ -197,14 +197,14 @@ pub fn list_files_recursive>(path: P) -> Result> { /// # Example /// /// ```rust,no_run -/// use filetools::sync::list_directories_recursive; +/// use filetools::sync::list_nested_directories; /// /// let target_dir = "some/dir/containing/nested/files"; /// /// // Will return a Vec containing all directories in the directory (including all subdirectories) -/// let dirs = list_directories_recursive(target_dir).expect("unable to list directories recursively"); +/// let dirs = list_nested_directories(target_dir).expect("unable to list directories recursively"); /// ``` -pub fn list_directories_recursive + Send>(path: P) -> Result> { +pub fn list_nested_directories + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), From 2ae2ecc3d2629d1166ec7689c0df1fd2bc3cbc3b Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Thu, 8 Feb 2024 20:01:29 +0000 Subject: [PATCH 085/105] added file filter search --- Cargo.toml | 1 + src/lib.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 683b784..934ce30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ readme = "README.md" anyhow = "1.0.79" async-recursion = "1.0.5" chrono = "0.3" +regex = "1.10.3" tokio = { version = "1.35.1", features = ["fs"] } uuid = { version = "0.8.1", features = ["v4"]} diff --git a/src/lib.rs b/src/lib.rs index 6c5797a..409076d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ //! TODO: More Docs! use anyhow::{Context, Result}; use async_recursion::async_recursion; +use regex::Regex; use std::path::{Component, Path, PathBuf}; use tokio::fs; @@ -34,6 +35,13 @@ pub(crate) enum FtIterItemState { DirRec, } +#[derive(Debug)] +pub enum FtFilter { + Raw(String), + Path(PathBuf), + Regex(Regex), +} + /// Checks if a given pattern is considered a subdirectory of the given path /// /// # Example @@ -220,6 +228,47 @@ pub async fn list_files + Send>(path: P) -> Result> iteritems(path, FtIterItemState::FileNoRec).await } +pub async fn list_files_with_filter + Send>( + path: P, + pattern: FtFilter, +) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + let results = iteritems(path, FtIterItemState::FileNoRec) + .await? + .into_iter() + .filter_map(|item| match &pattern { + FtFilter::Raw(raw) => { + if path_contains(&item, raw) { + return Some(item); + } + + None + } + FtFilter::Path(filter_path) => { + if path_contains(&item, filter_path) { + return Some(item); + } + + None + } + FtFilter::Regex(re) => { + if re.is_match(item.to_str().unwrap()) { + return Some(item); + } + + None + } + }) + .collect(); + + Ok(results) +} + /// Lists all directories in the given directory (not including subdirectories). /// /// # Errors @@ -616,4 +665,28 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn files_filter() -> Result<()> { + let root = TempPath::new("filter_files").await?; + root.multi_file(vec!["first.rs", "second.rs", "third.js", "fourth.rb"]) + .await?; + + let mut filter = FtFilter::Raw("fourth".to_string()); + let mut result = list_files_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 1); + assert_eq!(result[0], root.path.join("fourth.rb")); + + filter = FtFilter::Path(PathBuf::from("third.js")); + result = list_files_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 1); + assert_eq!(result[0], root.path.join("third.js")); + + filter = FtFilter::Regex(Regex::new(r"(.*)\.rs").unwrap()); + result = list_files_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 2); + assert!(result.contains(&root.path.join("first.rs"))); + assert!(result.contains(&root.path.join("second.rs"))); + Ok(()) + } } From 86bb3431f79aa5758d19802d34a9d0f6fec210f5 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Thu, 8 Feb 2024 20:08:21 +0000 Subject: [PATCH 086/105] moved some stuff to a util file for the crate --- src/lib.rs | 100 ++++------------------------------------------------ src/util.rs | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 94 deletions(-) create mode 100644 src/util.rs diff --git a/src/lib.rs b/src/lib.rs index 409076d..f85e3e9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,32 +8,17 @@ //! * Generate names for files / directories //! //! TODO: More Docs! +pub mod naming; +pub mod sync; +pub(crate) mod util; + use anyhow::{Context, Result}; use async_recursion::async_recursion; use regex::Regex; use std::path::{Component, Path, PathBuf}; use tokio::fs; -pub mod naming; -pub mod sync; - -/// Determines the type of iteration performed by the `list_directories` and `list_files` functions -/// If the NoRec variation is used, only the current directory is considered -/// If the Rec variation is used, then ALL subdirectores are traversed -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) enum FtIterItemState { - /// Iterate files with no recursion - FileNoRec, - - /// Iterate files with recursion - FileRec, - - /// Iterate directories with no recursion - DirNoRec, - - /// Iterate directories with recursion - DirRec, -} +use util::FtIterItemState; #[derive(Debug)] pub enum FtFilter { @@ -416,80 +401,7 @@ mod tests { use super::*; use anyhow::{Context, Result}; use std::path::PathBuf; - - /// Helper for creating temp directories - /// - /// Tempfile _would_ work but I want nested dirs and easy ways to create - /// a series of files / folder quickly without worrying - /// A cheap knock-off of `Tempfile` but meh, this works kinda better for my use case - struct TempPath { - pub path: PathBuf, - } - - impl TempPath { - pub async fn new(p: impl AsRef) -> Result { - let root = std::env::temp_dir(); - let path = if p.as_ref().starts_with(&root) { - p.as_ref().to_path_buf() - } else { - root.join(p) - }; - - ensure_directory(&path).await?; - - Ok(Self { path }) - } - - pub async fn new_file(&self, name: impl AsRef) -> Result { - let p = self.path.join(name); - tokio::fs::File::create(&p).await?; - - Self::new(p).await - } - - pub async fn multi_file(&self, names: Vec>) -> Result<()> { - for name in names { - tokio::fs::File::create(&self.path.join(name)).await?; - } - - Ok(()) - } - - pub async fn new_folder(&self, name: impl AsRef) -> Result { - let p = self.path.join(name); - ensure_directory(&p).await?; - - Self::new(p).await - } - - pub async fn multi_folder(&self, names: Vec>) -> Result<()> { - for name in names { - ensure_directory(&self.path.join(name)).await?; - } - - Ok(()) - } - - pub async fn nest_folders(&self, subfolder_chain: Vec>) -> Result { - let mut dst_path = self.path.clone(); - for sf in subfolder_chain { - dst_path = dst_path.join(sf.as_ref()); - } - - ensure_directory(&dst_path).await?; - Self::new(dst_path).await - } - - pub fn join(&self, path: impl AsRef) -> impl AsRef { - self.path.join(path) - } - } - - impl Drop for TempPath { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.path); - } - } + use util::TempPath; #[tokio::test] // This is kind of redundant as it just wraps `tokio::fs::create_dir_all` diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..1704f2b --- /dev/null +++ b/src/util.rs @@ -0,0 +1,95 @@ +use crate::ensure_directory; +use anyhow::Result; +use std::path::{Path, PathBuf}; + +/// Determines the type of iteration performed by the `list_directories` and `list_files` functions +/// If the NoRec variation is used, only the current directory is considered +/// If the Rec variation is used, then ALL subdirectores are traversed +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum FtIterItemState { + /// Iterate files with no recursion + FileNoRec, + + /// Iterate files with recursion + FileRec, + + /// Iterate directories with no recursion + DirNoRec, + + /// Iterate directories with recursion + DirRec, +} + +/// Helper for creating temp directories +/// +/// Tempfile _would_ work but I want nested dirs and easy ways to create +/// a series of files / folder quickly without worrying +/// A cheap knock-off of `Tempfile` but meh, this works kinda better for my use case +pub(crate) struct TempPath { + pub path: PathBuf, +} + +impl TempPath { + pub async fn new(p: impl AsRef) -> Result { + let root = std::env::temp_dir(); + let path = if p.as_ref().starts_with(&root) { + p.as_ref().to_path_buf() + } else { + root.join(p) + }; + + ensure_directory(&path).await?; + + Ok(Self { path }) + } + + pub async fn new_file(&self, name: impl AsRef) -> Result { + let p = self.path.join(name); + tokio::fs::File::create(&p).await?; + + Self::new(p).await + } + + pub async fn multi_file(&self, names: Vec>) -> Result<()> { + for name in names { + tokio::fs::File::create(&self.path.join(name)).await?; + } + + Ok(()) + } + + pub async fn new_folder(&self, name: impl AsRef) -> Result { + let p = self.path.join(name); + ensure_directory(&p).await?; + + Self::new(p).await + } + + pub async fn multi_folder(&self, names: Vec>) -> Result<()> { + for name in names { + ensure_directory(&self.path.join(name)).await?; + } + + Ok(()) + } + + pub async fn nest_folders(&self, subfolder_chain: Vec>) -> Result { + let mut dst_path = self.path.clone(); + for sf in subfolder_chain { + dst_path = dst_path.join(sf.as_ref()); + } + + ensure_directory(&dst_path).await?; + Self::new(dst_path).await + } + + pub fn join(&self, path: impl AsRef) -> impl AsRef { + self.path.join(path) + } +} + +impl Drop for TempPath { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} From fc2aa92882899f49db6358924f0590175f4fef2a Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 11:28:12 +0000 Subject: [PATCH 087/105] added some docs to FtFilter --- src/lib.rs | 29 ++++++++++++++++++++++++++++- src/util.rs | 2 ++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index f85e3e9..7dc4cda 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,10 +20,34 @@ use tokio::fs; use util::FtIterItemState; +/// Filter types for listing files / directories +/// +/// # Example +/// +/// ```rust +/// use filetools::FtFilter; +/// use std::path::PathBuf; +/// use regex::Regex; +/// +/// // Use a raw String filter to match an item containing ".log" +/// let filter = FtFilter::Raw(".log".to_string()); +/// +/// // Use the Path filter to match paths that contain `sub/path/to/math` +/// let filter = FtFilter::Path(PathBuf::from("sub/path/to/match")); +/// +/// // Use a Regex filter to match all files ending with `.rs` +/// let re = Regex::new(r"(.*)\.rs").expect("unable to create regex"); +/// let filter = FtFilter::Regex(re); +/// ``` #[derive(Debug)] pub enum FtFilter { + /// Filter based on a raw String pattern Raw(String), + + /// Filter based on a PathBuf pattern Path(PathBuf), + + /// Filter based on a regex pattern Regex(Regex), } @@ -227,6 +251,9 @@ pub async fn list_files_with_filter + Send>( .await? .into_iter() .filter_map(|item| match &pattern { + // I know these are the same for Raw and Path + // but it complains when you try and use the | with match + // for this FtFilter::Raw(raw) => { if path_contains(&item, raw) { return Some(item); @@ -403,9 +430,9 @@ mod tests { use std::path::PathBuf; use util::TempPath; - #[tokio::test] // This is kind of redundant as it just wraps `tokio::fs::create_dir_all` // but yay for test coverage i suppose + #[tokio::test] async fn creates_a_directory() -> Result<()> { let tmp = std::env::temp_dir(); diff --git a/src/util.rs b/src/util.rs index 1704f2b..36ce5ac 100644 --- a/src/util.rs +++ b/src/util.rs @@ -29,6 +29,8 @@ pub(crate) struct TempPath { pub path: PathBuf, } +// This is only used in the test suite +#[allow(dead_code)] impl TempPath { pub async fn new(p: impl AsRef) -> Result { let root = std::env::temp_dir(); From a86c76b999fa56fa0b2af5d9c8a3c89fe24753c4 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 15:42:36 +0000 Subject: [PATCH 088/105] some docs for filter_items --- src/lib.rs | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7dc4cda..6dbff23 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -164,8 +164,7 @@ pub async fn create_multiple_directories( Ok(()) } -/// Creates a range of numeric folders in the given path starting from `start` -/// up to `end` (non-inclusive). +/// Creates a range of numeric folders in the given path /// /// Directories can be padded with X zeros using the `fill` parameter. /// @@ -237,6 +236,37 @@ pub async fn list_files + Send>(path: P) -> Result> iteritems(path, FtIterItemState::FileNoRec).await } +/// Lists files in a folder (not including subdirectories) matching a filter pattern. +/// +/// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. +/// +/// # Example +/// +/// ```rust,no_run +/// use regex::Regex; +/// use std::path::PathBuf; +/// use filetools::{list_files_with_filter, FtFilter}; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let root = "some/path/containing/files"; +/// +/// // List all files containing the phrase `log` +/// let mut filter = FtFilter::Raw("log".to_string()); +/// let mut results = list_files_with_filter(&root, filter).await?; +/// +/// // List all files containing the path segment `files/test` +/// filter = FtFilter::Path(PathBuf::from("files/test")); +/// results = list_files_with_filter(&root, filter).await?; +/// +/// // List all files ending with `.rs` +/// let re = Regex::new(r"(.*)\.rs").expect("unable to create regex"); +/// filter = FtFilter::Regex(re); +/// results = list_files_with_filter(&root, filter).await?; +/// +/// Ok(()) +/// } +/// ``` pub async fn list_files_with_filter + Send>( path: P, pattern: FtFilter, From adaaf873544775aab1b51410bf7beccb12d3b8ed Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 17:22:43 +0000 Subject: [PATCH 089/105] more docs and tests plus impl for filter nested files --- src/lib.rs | 254 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 192 insertions(+), 62 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6dbff23..2467dcc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -233,7 +233,42 @@ pub async fn list_files + Send>(path: P) -> Result> "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::FileNoRec).await + iteritems(path, FtIterItemState::FileNoRec, None).await +} + +/// Lists all files in a directory including ALL subdirectories +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The given path is a file and not a directory +/// * The given path does not exist +/// +/// # Example +/// +/// ```rust,no_run +/// use filetools::list_nested_files; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let target_folder = "directory/containing/nested/files"; +/// +/// // This will return a Vec of ALL files contained within the directory +/// // (including in all subdirectories) +/// let files = list_nested_files(target_folder).await?; +/// Ok(()) +/// } +/// ``` +#[async_recursion] +pub async fn list_nested_files + Send>(path: P) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems(path, FtIterItemState::FileRec, None).await } /// Lists files in a folder (not including subdirectories) matching a filter pattern. @@ -277,74 +312,54 @@ pub async fn list_files_with_filter + Send>( "path should be a directory, not a file" ); - let results = iteritems(path, FtIterItemState::FileNoRec) - .await? - .into_iter() - .filter_map(|item| match &pattern { - // I know these are the same for Raw and Path - // but it complains when you try and use the | with match - // for this - FtFilter::Raw(raw) => { - if path_contains(&item, raw) { - return Some(item); - } - - None - } - FtFilter::Path(filter_path) => { - if path_contains(&item, filter_path) { - return Some(item); - } - - None - } - FtFilter::Regex(re) => { - if re.is_match(item.to_str().unwrap()) { - return Some(item); - } - - None - } - }) - .collect(); - - Ok(results) + iteritems(path, FtIterItemState::FileNoRec, Some(&pattern)).await } -/// Lists all directories in the given directory (not including subdirectories). -/// -/// # Errors +/// Lists files in a folder (including ALL subdirectories) matching a filter pattern. /// -/// This function will return an error in the following situations: -/// -/// * The given path is a file and not a directory -/// * The given path does not exist +/// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. /// /// # Example /// /// ```rust,no_run -/// use filetools::list_directories; +/// use regex::Regex; +/// use std::path::PathBuf; +/// use filetools::{list_files_with_filter, FtFilter}; /// /// #[tokio::main] /// async fn main() -> anyhow::Result<()> { -/// let target_folder = "directory/containing/other/directories"; +/// let root = "some/path/containing/nested/folders/with/files"; +/// +/// // List all files containing the phrase `log` +/// let mut filter = FtFilter::Raw("log".to_string()); +/// let mut results = list_files_with_filter(&root, filter).await?; +/// +/// // List all files containing the path segment `files/test` +/// filter = FtFilter::Path(PathBuf::from("files/test")); +/// results = list_files_with_filter(&root, filter).await?; +/// +/// // List all files ending with `.rs` +/// let re = Regex::new(r"(.*)\.rs").expect("unable to create regex"); +/// filter = FtFilter::Regex(re); +/// results = list_files_with_filter(&root, filter).await?; /// -/// // Will return a Vec containing all directories in the folder -/// let directories = list_directories(target_folder).await?; /// Ok(()) /// } /// ``` -pub async fn list_directories + Send>(path: P) -> Result> { +pub async fn list_nested_files_with_filter + Send>( + path: P, + pattern: FtFilter, +) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::DirNoRec).await + iteritems(path, FtIterItemState::FileRec, Some(&pattern)).await } -/// Lists all files in a directory including ALL subdirectories +/// Lists all directories in the given directory (not including subdirectories). /// /// # Errors /// @@ -356,27 +371,25 @@ pub async fn list_directories + Send>(path: P) -> Result anyhow::Result<()> { -/// let target_folder = "directory/containing/nested/files"; +/// let target_folder = "directory/containing/other/directories"; /// -/// // This will return a Vec of ALL files contained within the directory -/// // (including in all subdirectories) -/// let files = list_nested_files(target_folder).await?; +/// // Will return a Vec containing all directories in the folder +/// let directories = list_directories(target_folder).await?; /// Ok(()) /// } /// ``` -#[async_recursion] -pub async fn list_nested_files + Send>(path: P) -> Result> { +pub async fn list_directories + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::FileRec).await + iteritems(path, FtIterItemState::DirNoRec, None).await } /// Lists all directories in a directory including ALL subdirectories @@ -405,7 +418,33 @@ pub async fn list_nested_files + Send>(path: P) -> Result + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); - iteritems(path, FtIterItemState::DirRec).await + iteritems(path, FtIterItemState::DirRec, None).await +} + +/// Helper function to determine if an path item is valid based on the supplied filter +fn matches_filter(item: impl AsRef, filter: &FtFilter) -> bool { + match filter { + // I know these are the same for Raw and Path + // but it complains when you try and use the | with match + // for this + FtFilter::Raw(raw) => { + if path_contains(&item, raw) { + return true; + } + } + FtFilter::Path(filter_path) => { + if path_contains(&item, filter_path) { + return true; + } + } + FtFilter::Regex(re) => { + if re.is_match(item.as_ref().to_str().unwrap()) { + return true; + } + } + } + + false } /// Helper function to iterate through a directory to find all Files / Directories @@ -414,6 +453,7 @@ pub async fn list_nested_directories + Send>(path: P) -> Result + Send>( path: P, iterstate: FtIterItemState, + filter: Option<&'async_recursion FtFilter>, ) -> Result> { let mut items = vec![]; @@ -423,28 +463,39 @@ async fn iteritems + Send>( while let Some(entry) = entries.next_entry().await? { let e_path = entry.path(); + + // If a filter is present, set the value to the result of the filter + // check, else default to true so always adds the value + let filter_pass = match filter.as_ref() { + Some(f) => matches_filter(&e_path, f), + None => true, + }; + match iterstate { FtIterItemState::FileNoRec => { - if e_path.is_file() { + if e_path.is_file() && filter_pass { items.push(e_path); } } FtIterItemState::FileRec => { - if e_path.is_file() { + if e_path.is_file() && filter_pass { items.push(e_path) } else if e_path.is_dir() { - items.extend(iteritems(e_path, iterstate).await?); + items.extend(iteritems(e_path, iterstate, filter).await?); } } FtIterItemState::DirNoRec => { - if e_path.is_dir() { + if e_path.is_dir() && filter_pass { items.push(e_path); } } FtIterItemState::DirRec => { if e_path.is_dir() { - items.push(e_path.clone()); - items.extend(iteritems(e_path, iterstate).await?); + if filter_pass { + items.push(e_path.clone()); + } + + items.extend(iteritems(e_path, iterstate, filter).await?); } } } @@ -641,21 +692,100 @@ mod tests { root.multi_file(vec!["first.rs", "second.rs", "third.js", "fourth.rb"]) .await?; + // Raw string filter let mut filter = FtFilter::Raw("fourth".to_string()); let mut result = list_files_with_filter(&root.path, filter).await?; assert_eq!(result.len(), 1); assert_eq!(result[0], root.path.join("fourth.rb")); + // PathBuf filter filter = FtFilter::Path(PathBuf::from("third.js")); result = list_files_with_filter(&root.path, filter).await?; assert_eq!(result.len(), 1); assert_eq!(result[0], root.path.join("third.js")); + // Regex filter filter = FtFilter::Regex(Regex::new(r"(.*)\.rs").unwrap()); result = list_files_with_filter(&root.path, filter).await?; assert_eq!(result.len(), 2); assert!(result.contains(&root.path.join("first.rs"))); assert!(result.contains(&root.path.join("second.rs"))); + + Ok(()) + } + + #[tokio::test] + async fn files_filter_is_empty() -> Result<()> { + let root = TempPath::new("filter_files_empty").await?; + + // Raw string filter (normal + nested) + let mut filter = FtFilter::Raw("non-existant".to_string()); + let mut result = list_files_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + filter = FtFilter::Raw("non-existant".to_string()); + result = list_nested_files_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + + // PathBuf Filter + filter = FtFilter::Path(PathBuf::from("another-missing")); + result = list_files_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + filter = FtFilter::Path(PathBuf::from("another-missing")); + result = list_nested_files_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + + // Regex filter + filter = FtFilter::Regex(Regex::new(r"(.*)\.rs").unwrap()); + result = list_files_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + filter = FtFilter::Regex(Regex::new(r"(.*)\.rs").unwrap()); + result = list_nested_files_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn find_files_error() -> Result<()> { + let root = TempPath::new("filter_files_error").await?; + let test = root.new_file("test.js").await?; + + assert!(list_files(&test.path).await.is_err()); + assert!(list_nested_files(&test.path).await.is_err()); + assert!( + list_files_with_filter(&test.path, FtFilter::Raw("filter".to_string())) + .await + .is_err() + ); + assert!( + list_nested_files_with_filter(&test.path, FtFilter::Raw("filter".to_string())) + .await + .is_err() + ); + Ok(()) + } + + #[tokio::test] + async fn nested_files_filter() -> Result<()> { + let root = TempPath::new("nested_filter_files").await?; + let ffolder = root.new_folder("ffolder").await?; + let sfolder = root.new_folder("sfolder").await?; + let tfolder = root.new_folder("tfolder").await?; + + root.new_file("initial.pdf").await?; + ffolder.new_file("first.rs").await?; + sfolder.multi_file(vec!["second.txt", "third.rs"]).await?; + tfolder.new_file("initial.cpp").await?; + + let mut filter = FtFilter::Raw("initial".to_string()); + let mut result = list_nested_files_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 2); + assert!(result.contains(&root.path.join("tfolder/initial.cpp"))); + assert!(result.contains(&root.path.join("initial.pdf"))); + + filter = FtFilter::Path(PathBuf::from("second.txt")); + result = list_nested_files_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 1); + assert_eq!(result[0], root.path.join("sfolder/second.txt")); Ok(()) } } From cba5709646c642273354b78b6c95b975d5ca5d91 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 17:29:39 +0000 Subject: [PATCH 090/105] minor refactor of things --- src/lib.rs | 85 +------------------------------------------------- src/util.rs | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 86 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2467dcc..7f70a12 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,7 @@ use regex::Regex; use std::path::{Component, Path, PathBuf}; use tokio::fs; -use util::FtIterItemState; +use util::{iteritems, FtIterItemState}; /// Filter types for listing files / directories /// @@ -421,89 +421,6 @@ pub async fn list_nested_directories + Send>(path: P) -> Result, filter: &FtFilter) -> bool { - match filter { - // I know these are the same for Raw and Path - // but it complains when you try and use the | with match - // for this - FtFilter::Raw(raw) => { - if path_contains(&item, raw) { - return true; - } - } - FtFilter::Path(filter_path) => { - if path_contains(&item, filter_path) { - return true; - } - } - FtFilter::Regex(re) => { - if re.is_match(item.as_ref().to_str().unwrap()) { - return true; - } - } - } - - false -} - -/// Helper function to iterate through a directory to find all Files / Directories -/// depending on the `FilterState` passed. -#[async_recursion] -async fn iteritems + Send>( - path: P, - iterstate: FtIterItemState, - filter: Option<&'async_recursion FtFilter>, -) -> Result> { - let mut items = vec![]; - - let mut entries = fs::read_dir(path.as_ref()) - .await - .context("list items inner call")?; - - while let Some(entry) = entries.next_entry().await? { - let e_path = entry.path(); - - // If a filter is present, set the value to the result of the filter - // check, else default to true so always adds the value - let filter_pass = match filter.as_ref() { - Some(f) => matches_filter(&e_path, f), - None => true, - }; - - match iterstate { - FtIterItemState::FileNoRec => { - if e_path.is_file() && filter_pass { - items.push(e_path); - } - } - FtIterItemState::FileRec => { - if e_path.is_file() && filter_pass { - items.push(e_path) - } else if e_path.is_dir() { - items.extend(iteritems(e_path, iterstate, filter).await?); - } - } - FtIterItemState::DirNoRec => { - if e_path.is_dir() && filter_pass { - items.push(e_path); - } - } - FtIterItemState::DirRec => { - if e_path.is_dir() { - if filter_pass { - items.push(e_path.clone()); - } - - items.extend(iteritems(e_path, iterstate, filter).await?); - } - } - } - } - - Ok(items) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/util.rs b/src/util.rs index 36ce5ac..adcab84 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,6 +1,8 @@ -use crate::ensure_directory; -use anyhow::Result; +use crate::{ensure_directory, path_contains, FtFilter}; +use anyhow::{Context, Result}; +use async_recursion::async_recursion; use std::path::{Path, PathBuf}; +use tokio::fs; /// Determines the type of iteration performed by the `list_directories` and `list_files` functions /// If the NoRec variation is used, only the current directory is considered @@ -20,6 +22,89 @@ pub(crate) enum FtIterItemState { DirRec, } +/// Helper function to determine if an path item is valid based on the supplied filter +fn matches_filter(item: impl AsRef, filter: &FtFilter) -> bool { + match filter { + // I know these are the same for Raw and Path + // but it complains when you try and use the | with match + // for this + FtFilter::Raw(raw) => { + if path_contains(&item, raw) { + return true; + } + } + FtFilter::Path(filter_path) => { + if path_contains(&item, filter_path) { + return true; + } + } + FtFilter::Regex(re) => { + if re.is_match(item.as_ref().to_str().unwrap()) { + return true; + } + } + } + + false +} + +/// Helper function to iterate through a directory to find all Files / Directories +/// depending on the `FilterState` passed. +#[async_recursion] +pub(crate) async fn iteritems + Send>( + path: P, + iterstate: FtIterItemState, + filter: Option<&'async_recursion FtFilter>, +) -> Result> { + let mut items = vec![]; + + let mut entries = fs::read_dir(path.as_ref()) + .await + .context("list items inner call")?; + + while let Some(entry) = entries.next_entry().await? { + let e_path = entry.path(); + + // If a filter is present, set the value to the result of the filter + // check, else default to true so always adds the value + let filter_pass = match filter.as_ref() { + Some(f) => matches_filter(&e_path, f), + None => true, + }; + + match iterstate { + FtIterItemState::FileNoRec => { + if e_path.is_file() && filter_pass { + items.push(e_path); + } + } + FtIterItemState::FileRec => { + if e_path.is_file() && filter_pass { + items.push(e_path) + } else if e_path.is_dir() { + items.extend(iteritems(e_path, iterstate, filter).await?); + } + } + FtIterItemState::DirNoRec => { + if e_path.is_dir() && filter_pass { + items.push(e_path); + } + } + FtIterItemState::DirRec => { + if e_path.is_dir() { + if filter_pass { + items.push(e_path.clone()); + } + + items.extend(iteritems(e_path, iterstate, filter).await?); + } + } + } + } + + Ok(items) +} + /// Helper for creating temp directories /// /// Tempfile _would_ work but I want nested dirs and easy ways to create From 7caddeada3293a5eae99d89a3b03515f2dc81f14 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 17:49:40 +0000 Subject: [PATCH 091/105] added dirs_filter functions --- src/lib.rs | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7f70a12..9d2a14c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,6 @@ pub mod sync; pub(crate) mod util; use anyhow::{Context, Result}; -use async_recursion::async_recursion; use regex::Regex; use std::path::{Component, Path, PathBuf}; use tokio::fs; @@ -260,7 +259,6 @@ pub async fn list_files + Send>(path: P) -> Result> /// Ok(()) /// } /// ``` -#[async_recursion] pub async fn list_nested_files + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( @@ -415,12 +413,99 @@ pub async fn list_directories + Send>(path: P) -> Result + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); iteritems(path, FtIterItemState::DirRec, None).await } +/// Lists directories in a given directory (not including subdirectories) matching a filter pattern. +/// +/// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. +/// +/// # Example +/// +/// ```rust,no_run +/// use regex::Regex; +/// use std::path::PathBuf; +/// use filetools::{list_directories_with_filter, FtFilter}; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let root = "some/path/containing/dirs"; +/// +/// // List all dirs containing the phrase `log` +/// let mut filter = FtFilter::Raw("log".to_string()); +/// let mut results = list_directories_with_filter(&root, filter).await?; +/// +/// // List all dirs containing the path segment `files/test` +/// filter = FtFilter::Path(PathBuf::from("files/test")); +/// results = list_directories_with_filter(&root, filter).await?; +/// +/// // List all dirs ending with `_test` +/// let re = Regex::new(r"(.*)_test").expect("unable to create regex"); +/// filter = FtFilter::Regex(re); +/// results = list_directories_with_filter(&root, filter).await?; +/// +/// Ok(()) +/// } +/// ``` +pub async fn list_directories_with_filter + Send>( + path: P, + filter: FtFilter, +) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems(path, FtIterItemState::DirNoRec, Some(&filter)).await +} + +/// Lists directories in a given directory (including ALL subdirectories) matching a filter pattern. +/// +/// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. +/// +/// # Example +/// +/// ```rust,no_run +/// use regex::Regex; +/// use std::path::PathBuf; +/// use filetools::{list_directories_with_filter, FtFilter}; +/// +/// #[tokio::main] +/// async fn main() -> anyhow::Result<()> { +/// let root = "some/path/containing/dirs"; +/// +/// // List all dirs containing the phrase `log` +/// let mut filter = FtFilter::Raw("log".to_string()); +/// let mut results = list_directories_with_filter(&root, filter).await?; +/// +/// // List all dirs containing the path segment `files/test` +/// filter = FtFilter::Path(PathBuf::from("files/test")); +/// results = list_directories_with_filter(&root, filter).await?; +/// +/// // List all dirs ending with `_test` +/// let re = Regex::new(r"(.*)_test").expect("unable to create regex"); +/// filter = FtFilter::Regex(re); +/// results = list_directories_with_filter(&root, filter).await?; +/// +/// Ok(()) +/// } +/// ``` +pub async fn list_nested_directories_with_filter + Send>( + path: P, + filter: FtFilter, +) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems(path, FtIterItemState::DirRec, Some(&filter)).await +} + #[cfg(test)] mod tests { use super::*; From 8a26d42ca16a4ef37228ea3b880b8544669d3d9c Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 18:16:15 +0000 Subject: [PATCH 092/105] tests for dirs with filters --- src/lib.rs | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 9d2a14c..0f0244a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -747,7 +747,7 @@ mod tests { } #[tokio::test] - async fn find_files_error() -> Result<()> { + async fn list_items_error() -> Result<()> { let root = TempPath::new("filter_files_error").await?; let test = root.new_file("test.js").await?; @@ -763,6 +763,19 @@ mod tests { .await .is_err() ); + assert!(list_directories(&test.path).await.is_err()); + assert!(list_nested_directories(&test.path).await.is_err()); + assert!( + list_directories_with_filter(&test.path, FtFilter::Raw("filter".to_string())) + .await + .is_err() + ); + assert!(list_nested_directories_with_filter( + &test.path, + FtFilter::Raw("filter".to_string()) + ) + .await + .is_err()); Ok(()) } @@ -790,4 +803,91 @@ mod tests { assert_eq!(result[0], root.path.join("sfolder/second.txt")); Ok(()) } + + #[tokio::test] + async fn directories_filter() -> Result<()> { + let root = TempPath::new("dir_filter").await?; + root.multi_folder(vec!["log_var", "store_var", "config", "etc"]) + .await?; + + // Raw string filter + let mut filter = FtFilter::Raw("config".to_string()); + let mut result = list_directories_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 1); + assert_eq!(result[0], root.path.join("config")); + + // PathBuf filter + filter = FtFilter::Path(PathBuf::from("etc")); + result = list_directories_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 1); + assert_eq!(result[0], root.path.join("etc")); + + // Regex filter + filter = FtFilter::Regex(Regex::new(r"(.*)_var").unwrap()); + result = list_directories_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 2); + assert!(result.contains(&root.path.join("log_var"))); + assert!(result.contains(&root.path.join("store_var"))); + Ok(()) + } + + #[tokio::test] + async fn nested_directories_filter() -> Result<()> { + let root = TempPath::new("nested_dir_filter_test").await?; + root.multi_folder(vec!["folder1", "folder2"]).await?; + + let f1 = TempPath::new(root.join("folder1")).await?; + f1.multi_folder(vec!["sub1", "sub_2", "sub3"]).await?; + + let s2 = TempPath::new(f1.join("sub_2")).await?; + s2.multi_folder(vec!["deep_1", "deep2"]).await?; + + // Raw filter + let mut filter = FtFilter::Raw("deep".to_string()); + let mut result = list_nested_directories_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 2); + assert!(result.contains(&root.path.join("folder1/sub_2/deep_1"))); + assert!(result.contains(&root.path.join("folder1/sub_2/deep2"))); + + // Path filter + filter = FtFilter::Path(PathBuf::from("folder1")); + result = list_nested_directories_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 6); + + filter = FtFilter::Regex(Regex::new(r"(.*)_[0-9]{1}").unwrap()); + result = list_nested_directories_with_filter(&root.path, filter).await?; + assert_eq!(result.len(), 3); + + Ok(()) + } + + #[tokio::test] + async fn list_dirs_empty() -> Result<()> { + let root = TempPath::new("list_dirs_empty").await?; + + // Raw string filter (normal + nested) + let mut filter = FtFilter::Raw("non-existant".to_string()); + let mut result = list_directories_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + filter = FtFilter::Raw("non-existant".to_string()); + result = list_nested_directories_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + + // PathBuf Filter + filter = FtFilter::Path(PathBuf::from("another-missing")); + result = list_directories_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + filter = FtFilter::Path(PathBuf::from("another-missing")); + result = list_nested_directories_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + + // Regex filter + filter = FtFilter::Regex(Regex::new(r"(.*)\.rs").unwrap()); + result = list_directories_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + filter = FtFilter::Regex(Regex::new(r"(.*)\.rs").unwrap()); + result = list_nested_directories_with_filter(&root.path, filter).await?; + assert!(result.is_empty()); + Ok(()) + } } From f3ebb4c60ecc29dcee23202e56c02edcd0729429 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 18:34:58 +0000 Subject: [PATCH 093/105] adding sync version of filter functions --- src/lib.rs | 6 ++++++ src/sync.rs | 48 ++++++------------------------------------------ src/util.rs | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0f0244a..0f74a19 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,8 +5,14 @@ //! * Create directories //! * Check filepaths contain a pattern //! * List files / directories both iteratively and recursively +//! * List files / directories both iteratively and recursively with filters //! * Generate names for files / directories //! +//! ## Async vs Sync +//! +//! The operations in this crate are designed for async/await, however sync variations +//! of the operations exist in the [`crate::sync`] module. +//! //! TODO: More Docs! pub mod naming; pub mod sync; diff --git a/src/sync.rs b/src/sync.rs index 1d6f3d1..1c5f077 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,6 @@ //! Sync variations of the main [`crate`] functions -use crate::{naming::generate_n_digit_name, FtIterItemState}; +use crate::util::FtIterItemState; +use crate::{naming::generate_n_digit_name, util::iteritems_sync, FtFilter}; use anyhow::{Context, Result}; use std::fs; use std::path::{Path, PathBuf}; @@ -119,7 +120,7 @@ pub fn list_files>(path: P) -> Result>> { "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::FileNoRec) + iteritems_sync(path, FtIterItemState::FileNoRec, None) } /// Lists all directories in the given directory (not including subdirectories). @@ -149,7 +150,7 @@ pub fn list_directories>(path: P) -> Result> path.as_ref().is_dir(), "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::DirNoRec) + iteritems_sync(path, FtIterItemState::DirNoRec, None) } /// Lists all files in a directory including ALL subdirectories @@ -180,7 +181,7 @@ pub fn list_nested_files>(path: P) -> Result> { "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::FileRec) + iteritems_sync(path, FtIterItemState::FileRec, None) } /// Lists all directories in a directory including ALL subdirectories @@ -210,44 +211,7 @@ pub fn list_nested_directories + Send>(path: P) -> Result>(path: P, iterstate: FtIterItemState) -> Result> { - let mut items = vec![]; - - let mut entries = fs::read_dir(path.as_ref()).context("list items inner call")?; - - while let Some(Ok(entry)) = entries.next() { - let e_path = entry.path(); - match iterstate { - FtIterItemState::FileNoRec => { - if e_path.is_file() { - items.push(e_path); - } - } - FtIterItemState::FileRec => { - if e_path.is_file() { - items.push(e_path) - } else if e_path.is_dir() { - items.extend(iteritems(e_path, iterstate)?); - } - } - FtIterItemState::DirNoRec => { - if e_path.is_dir() { - items.push(e_path); - } - } - FtIterItemState::DirRec => { - if e_path.is_dir() { - items.push(e_path.clone()); - items.extend(iteritems(e_path, iterstate)?); - } - } - } - } - - Ok(items) + iteritems_sync(path, FtIterItemState::DirRec, None) } // Do I write tests for these as they are just the sync version of the Ft functions diff --git a/src/util.rs b/src/util.rs index adcab84..6b76850 100644 --- a/src/util.rs +++ b/src/util.rs @@ -105,6 +105,56 @@ pub(crate) async fn iteritems + Send>( Ok(items) } +pub(crate) fn iteritems_sync>( + path: P, + iterstate: FtIterItemState, + filter: Option<&FtFilter>, +) -> Result> { + let mut items = vec![]; + + let mut entries = std::fs::read_dir(path.as_ref()).context("sync iteritems entry call")?; + + while let Some(Ok(entry)) = entries.next() { + let e_path = entry.path(); + + // If a filter is present, set the value to the result of the filter + // check, else default to true so always adds the value + let filter_pass = match filter.as_ref() { + Some(f) => matches_filter(&e_path, f), + None => true, + }; + match iterstate { + FtIterItemState::FileNoRec => { + if e_path.is_file() && filter_pass { + items.push(e_path); + } + } + FtIterItemState::FileRec => { + if e_path.is_file() && filter_pass { + items.push(e_path) + } else if e_path.is_dir() { + items.extend(iteritems_sync(e_path, iterstate, filter)?); + } + } + FtIterItemState::DirNoRec => { + if e_path.is_dir() && filter_pass { + items.push(e_path); + } + } + FtIterItemState::DirRec => { + if e_path.is_dir() { + if filter_pass { + items.push(e_path.clone()); + } + + items.extend(iteritems_sync(e_path, iterstate, filter)?); + } + } + } + } + + Ok(items) +} /// Helper for creating temp directories /// /// Tempfile _would_ work but I want nested dirs and easy ways to create From ff917099767590432f9671b66dadccb2e543aa37 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Fri, 9 Feb 2024 19:08:16 +0000 Subject: [PATCH 094/105] loads of docs added --- src/lib.rs | 90 ++++++++++++++++++-- src/sync.rs | 238 +++++++++++++++++++++++++++++++++++++++++++++------- src/util.rs | 2 + 3 files changed, 293 insertions(+), 37 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0f74a19..e07b2c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -113,6 +113,10 @@ pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe /// /// If the directory already exists, nothing is done /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::ensure_directory`] +/// /// # Example /// /// ```rust,no_run @@ -138,6 +142,10 @@ pub async fn ensure_directory(dir: impl AsRef) -> Result<()> { /// Creates multiple directories inside the target path. /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::create_multiple_directories`] +/// /// # Example /// /// ```rust,no_run @@ -173,6 +181,10 @@ pub async fn create_multiple_directories( /// /// Directories can be padded with X zeros using the `fill` parameter. /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::create_numeric_directories`] +/// /// # Example /// /// ```rust,no_run @@ -210,6 +222,10 @@ pub async fn create_numeric_directories( /// Lists all files in the given directory (not including subdirectories). /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::list_files`] +/// /// # Errors /// /// This function will return an error in the following situations: @@ -217,6 +233,7 @@ pub async fn create_numeric_directories( /// * The path given is a file and not a directory /// * The given path does not exist /// +/// /// # Example /// /// ```rust,no_run @@ -243,6 +260,10 @@ pub async fn list_files + Send>(path: P) -> Result> /// Lists all files in a directory including ALL subdirectories /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::list_nested_files`] +/// /// # Errors /// /// This function will return an error in the following situations: @@ -279,6 +300,17 @@ pub async fn list_nested_files + Send>(path: P) -> Result + Send>( /// /// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::list_nested_files_with_filter`] +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The given path is a file and not a directory +/// * The given path does not exist +/// /// # Example /// /// ```rust,no_run /// use regex::Regex; /// use std::path::PathBuf; -/// use filetools::{list_files_with_filter, FtFilter}; +/// use filetools::{list_nested_files_with_filter, FtFilter}; /// /// #[tokio::main] /// async fn main() -> anyhow::Result<()> { @@ -336,16 +379,16 @@ pub async fn list_files_with_filter + Send>( /// /// // List all files containing the phrase `log` /// let mut filter = FtFilter::Raw("log".to_string()); -/// let mut results = list_files_with_filter(&root, filter).await?; +/// let mut results = list_nested_files_with_filter(&root, filter).await?; /// /// // List all files containing the path segment `files/test` /// filter = FtFilter::Path(PathBuf::from("files/test")); -/// results = list_files_with_filter(&root, filter).await?; +/// results = list_nested_files_with_filter(&root, filter).await?; /// /// // List all files ending with `.rs` /// let re = Regex::new(r"(.*)\.rs").expect("unable to create regex"); /// filter = FtFilter::Regex(re); -/// results = list_files_with_filter(&root, filter).await?; +/// results = list_nested_files_with_filter(&root, filter).await?; /// /// Ok(()) /// } @@ -365,6 +408,10 @@ pub async fn list_nested_files_with_filter + Send>( /// Lists all directories in the given directory (not including subdirectories). /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::list_directories`] +/// /// # Errors /// /// This function will return an error in the following situations: @@ -398,10 +445,15 @@ pub async fn list_directories + Send>(path: P) -> Result + Send>(path: P) -> Result + Send>( /// /// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. /// +/// ## Sync +/// +/// For the `sync` version, see [`crate::sync::list_nested_directories_with_filter`] +/// +/// # Errors +/// +/// This function will return an error in the following situations: +/// +/// * The given path is a file and not a directory +/// * The given path does not exist +/// /// # Example /// /// ```rust,no_run /// use regex::Regex; /// use std::path::PathBuf; -/// use filetools::{list_directories_with_filter, FtFilter}; +/// use filetools::{list_nested_directories_with_filter, FtFilter}; /// /// #[tokio::main] /// async fn main() -> anyhow::Result<()> { @@ -485,16 +559,16 @@ pub async fn list_directories_with_filter + Send>( /// /// // List all dirs containing the phrase `log` /// let mut filter = FtFilter::Raw("log".to_string()); -/// let mut results = list_directories_with_filter(&root, filter).await?; +/// let mut results = list_nested_directories_with_filter(&root, filter).await?; /// /// // List all dirs containing the path segment `files/test` /// filter = FtFilter::Path(PathBuf::from("files/test")); -/// results = list_directories_with_filter(&root, filter).await?; +/// results = list_nested_directories_with_filter(&root, filter).await?; /// /// // List all dirs ending with `_test` /// let re = Regex::new(r"(.*)_test").expect("unable to create regex"); /// filter = FtFilter::Regex(re); -/// results = list_directories_with_filter(&root, filter).await?; +/// results = list_nested_directories_with_filter(&root, filter).await?; /// /// Ok(()) /// } diff --git a/src/sync.rs b/src/sync.rs index 1c5f077..9a8aed6 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -9,7 +9,9 @@ use std::path::{Path, PathBuf}; /// /// If the directory already exists, nothing is done /// -/// This is the sync version of [`crate::ensure_directory`] +/// ## Async +/// +/// For the `async` version, see: [`crate::ensure_directory`] /// /// # Example /// @@ -28,12 +30,13 @@ pub fn ensure_directory(dir: impl AsRef) -> Result<()> { Ok(()) } -/// Creates a range of numeric folders in the given path starting from `start` -/// up to `end` (non-inclusive). +/// Creates a range of numeric folders in the given path /// /// Directories can be padded with X zeros using the `fill` parameter. /// -/// This is the sync version of [`crate::create_numeric_directories`] +/// ## Async +/// +/// For the `async` version, see: [`crate::create_numeric_directories`] /// /// # Example /// @@ -64,7 +67,9 @@ pub fn create_numeric_directories( /// Creates multiple directories inside the target path. /// -/// This is the sync version of [`crate::create_multiple_directories`] +/// ## Async +/// +/// For the `async` version, see: [`crate::create_multiple_directories`] /// /// # Example /// @@ -94,7 +99,9 @@ pub fn create_multiple_directories( /// Lists all files in the given directory (not including subdirectories). /// -/// This is the sync version of [`crate::list_files`] +/// ## Async +/// +/// For the `async` version, see: [`crate::list_files`] /// /// # Errors /// @@ -123,70 +130,161 @@ pub fn list_files>(path: P) -> Result>> { iteritems_sync(path, FtIterItemState::FileNoRec, None) } -/// Lists all directories in the given directory (not including subdirectories). +/// Lists all files in a directory including ALL subdirectories /// -/// This is the sync version of [`crate::list_directories`] +/// ## Async +/// +/// For the `async` version, see: [`crate::list_nested_files`] /// /// # Errors /// /// This function will return an error in the following situations: /// -/// * The path given is a file and not a directory +/// * The given path is a file and not a directory /// * The given path does not exist /// /// # Example /// /// ```rust,no_run -/// use filetools::sync::list_directories; +/// use filetools::sync::list_nested_files; /// -/// let target_dir = "some/dir/containing/files"; +/// let target_dir = "some/dir/containing/nested/files"; /// -/// // Will return a Vec containing paths to all directories in the directory -/// let dirs = list_directories(target_dir).expect("unable to list directories"); +/// // Will return a Vec containing all files in the directory (including all subdirectories) +/// let files = list_nested_files(target_dir).expect("unable to list files recursively"); /// ``` -pub fn list_directories>(path: P) -> Result>> { +pub fn list_nested_files>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::DirNoRec, None) + + iteritems_sync(path, FtIterItemState::FileRec, None) } -/// Lists all files in a directory including ALL subdirectories +/// Lists files in a folder (not including subdirectories) matching a filter pattern. +/// +/// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. +/// +/// ## Async +/// +/// For the `async` version, see: [`crate::list_files_with_filter`] +/// +/// # Example +/// +/// ```rust,no_run +/// use regex::Regex; +/// use std::path::PathBuf; +/// use filetools::{sync::list_files_with_filter, FtFilter}; +/// +/// let root = "some/path/containing/files"; /// -/// This is the sync version of [`crate::list_nested_files`] +/// // List all files containing the phrase `log` +/// let mut filter = FtFilter::Raw("log".to_string()); +/// let mut results = list_files_with_filter(&root, filter).expect("unable to list filtered files"); +/// +/// // List all files containing the path segment `files/test` +/// filter = FtFilter::Path(PathBuf::from("files/test")); +/// results = list_files_with_filter(&root, filter).expect("unable to list filtered files"); +/// +/// // List all files ending with `.rs` +/// let re = Regex::new(r"(.*)\.rs").expect("unable to create regex"); +/// filter = FtFilter::Regex(re); +/// results = list_files_with_filter(&root, filter).expect("unable to list filtered files"); +/// +/// +/// ``` +pub fn list_files_with_filter>(path: P, filter: FtFilter) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems_sync(path, FtIterItemState::FileNoRec, Some(&filter)) +} + +/// Lists files in a folder (including ALL subdirectories) matching a filter pattern. +/// +/// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. +/// +/// ## Async +/// +/// For the `async` version, see: [`crate::list_nested_files_with_filter`] +/// +/// # Example +/// +/// ```rust,no_run +/// use regex::Regex; +/// use std::path::PathBuf; +/// use filetools::{sync::list_nested_files_with_filter, FtFilter}; +/// +/// let root = "some/path/containing/nested/folders/with/files"; +/// +/// // List all files containing the phrase `log` +/// let mut filter = FtFilter::Raw("log".to_string()); +/// let mut results = list_nested_files_with_filter(&root, filter).expect("unable to list nested files with filter"); +/// +/// // List all files containing the path segment `files/test` +/// filter = FtFilter::Path(PathBuf::from("files/test")); +/// results = list_nested_files_with_filter(&root, filter).expect("unable to list nested files with filter"); +/// +/// // List all files ending with `.rs` +/// let re = Regex::new(r"(.*)\.rs").expect("unable to create regex"); +/// filter = FtFilter::Regex(re); +/// results = list_nested_files_with_filter(&root, filter).expect("unable to list nested files with filter"); +/// ``` +pub fn list_nested_files_with_filter>( + path: P, + filter: FtFilter, +) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + + iteritems_sync(path, FtIterItemState::FileRec, Some(&filter)) +} + +/// Lists all directories in the given directory (not including subdirectories). +/// +/// ## Async +/// +/// For the `async` version, see: [`crate::list_directories`] /// /// # Errors /// /// This function will return an error in the following situations: /// -/// * The given path is a file and not a directory +/// * The path given is a file and not a directory /// * The given path does not exist /// /// # Example /// /// ```rust,no_run -/// use filetools::sync::list_nested_files; +/// use filetools::sync::list_directories; /// -/// let target_dir = "some/dir/containing/nested/files"; +/// let target_dir = "some/dir/containing/files"; /// -/// // Will return a Vec containing all files in the directory (including all subdirectories) -/// let files = list_nested_files(target_dir).expect("unable to list files recursively"); +/// // Will return a Vec containing paths to all directories in the directory +/// let dirs = list_directories(target_dir).expect("unable to list directories"); /// ``` -pub fn list_nested_files>(path: P) -> Result> { +pub fn list_directories>(path: P) -> Result>> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); anyhow::ensure!( path.as_ref().is_dir(), "path should be a directory, not a file" ); - - iteritems_sync(path, FtIterItemState::FileRec, None) + iteritems_sync(path, FtIterItemState::DirNoRec, None) } /// Lists all directories in a directory including ALL subdirectories /// -/// This is the sync version of [`crate::list_nested_directories`] +/// ## Async +/// +/// For the `async` version, see: [`crate::list_nested_directories`] /// /// # Errors /// @@ -214,6 +312,88 @@ pub fn list_nested_directories + Send>(path: P) -> Result>( + path: P, + filter: FtFilter, +) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + iteritems_sync(path, FtIterItemState::DirNoRec, Some(&filter)) +} + +/// Lists directories in a given directory (including ALL subdirectories) matching a filter pattern. +/// +/// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. +/// +/// ## Async +/// +/// For the `async` version, see: [`crate::list_nested_directories_with_filter`] +/// +/// # Example +/// +/// ```rust,no_run +/// use regex::Regex; +/// use std::path::PathBuf; +/// use filetools::{sync::list_nested_directories_with_filter, FtFilter}; +/// +/// let root = "some/path/containing/dirs"; +/// +/// // List all dirs containing the phrase `log` +/// let mut filter = FtFilter::Raw("log".to_string()); +/// let mut results = list_nested_directories_with_filter(&root, filter).expect("unable to list nested dirs with filter"); +/// +/// // List all dirs containing the path segment `files/test` +/// filter = FtFilter::Path(PathBuf::from("files/test")); +/// results = list_nested_directories_with_filter(&root, filter).expect("unable to list nested dirs with filter"); +/// +/// // List all dirs ending with `_test` +/// let re = Regex::new(r"(.*)_test").expect("unable to create regex"); +/// filter = FtFilter::Regex(re); +/// results = list_nested_directories_with_filter(&root, filter).expect("unable to list nested dirs with filter"); +/// ``` +pub fn list_nested_directories_with_filter>( + path: P, + filter: FtFilter, +) -> Result> { + anyhow::ensure!(path.as_ref().exists(), "path does not exist"); + anyhow::ensure!( + path.as_ref().is_dir(), + "path should be a directory, not a file" + ); + iteritems_sync(path, FtIterItemState::DirRec, Some(&filter)) +} + +// No tests needed cause these are tested in the main crate diff --git a/src/util.rs b/src/util.rs index 6b76850..363d0f0 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,3 +1,5 @@ +//! Internal helper utilities and types + use crate::{ensure_directory, path_contains, FtFilter}; use anyhow::{Context, Result}; use async_recursion::async_recursion; From 9ed464522522f4bed17fcb25309ec75c7c7bb10d Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 17:53:32 +0000 Subject: [PATCH 095/105] docs: more doc updates --- .gitignore | 1 - Cargo.lock | 352 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 17 ++- src/lib.rs | 11 +- src/sync.rs | 10 ++ 5 files changed, 386 insertions(+), 5 deletions(-) create mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index 38da3ee..0176326 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ /target -Cargo.lock *test* diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..b6841db --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,352 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" + +[[package]] +name = "async-recursion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "158b0bd7d75cbb6bf9c25967a48a2e9f77da95876b858eadfabaa99cd069de6e" +dependencies = [ + "num", + "time", +] + +[[package]] +name = "filetools" +version = "0.3.0" +dependencies = [ + "anyhow", + "async-recursion", + "chrono", + "regex", + "tokio", + "uuid", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "hermit-abi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "num" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" +dependencies = [ + "num-integer", + "num-iter", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "syn" +version = "2.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "tokio" +version = "1.35.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +dependencies = [ + "backtrace", + "num_cpus", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/README.md b/README.md index 29afb83..b0b2f16 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,23 @@ # Filetools -Simple port of the filetools library written by Jonathan Grizou here: https://github.com/jgrizou/filetools -This library allows for simple filename generation and finding all files/folders in a directory. +Simple crate for perfoming some small `Path` operations in Rust. + +Offers the user the ability to: + +* Create directories (single / multiple at a time) +* Check given filepaths match a pattern +* List all files / directories in a path + * This can be just the files / directories inside the path root + * This can also include files / directories in **ALL** subdirectories contained in the path +* List files / directories as above but filter the results based on a Filter Pattern +* Some general naming functions for creating `PathBuf` names + +More will be added in the future but this should suffice for small path operations. ## Usage + Add to your `Cargo.toml` + ```toml [dependencies] filetools = "0.2.0" diff --git a/src/lib.rs b/src/lib.rs index e07b2c3..f40db6e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,8 +12,7 @@ //! //! The operations in this crate are designed for async/await, however sync variations //! of the operations exist in the [`crate::sync`] module. -//! -//! TODO: More Docs! + pub mod naming; pub mod sync; pub(crate) mod util; @@ -260,6 +259,8 @@ pub async fn list_files + Send>(path: P) -> Result> /// Lists all files in a directory including ALL subdirectories /// +/// Use responsibly. +/// /// ## Sync /// /// For the `sync` version, see [`crate::sync::list_nested_files`] @@ -355,6 +356,8 @@ pub async fn list_files_with_filter + Send>( /// /// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. /// +/// Use responsibly. +/// /// ## Sync /// /// For the `sync` version, see [`crate::sync::list_nested_files_with_filter`] @@ -445,6 +448,8 @@ pub async fn list_directories + Send>(path: P) -> Result + Send>( /// /// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. /// +/// Use responsibly. +/// /// ## Sync /// /// For the `sync` version, see [`crate::sync::list_nested_directories_with_filter`] diff --git a/src/sync.rs b/src/sync.rs index 9a8aed6..87e4ca4 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,4 +1,6 @@ //! Sync variations of the main [`crate`] functions +//! +//! All operations are identical to those defined in the `async` version. use crate::util::FtIterItemState; use crate::{naming::generate_n_digit_name, util::iteritems_sync, FtFilter}; use anyhow::{Context, Result}; @@ -132,6 +134,8 @@ pub fn list_files>(path: P) -> Result>> { /// Lists all files in a directory including ALL subdirectories /// +/// Use responsibly. +/// /// ## Async /// /// For the `async` version, see: [`crate::list_nested_files`] @@ -209,6 +213,8 @@ pub fn list_files_with_filter>(path: P, filter: FtFilter) -> Resu /// /// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. /// +/// Use responsibly. +/// /// ## Async /// /// For the `async` version, see: [`crate::list_nested_files_with_filter`] @@ -282,6 +288,8 @@ pub fn list_directories>(path: P) -> Result> /// Lists all directories in a directory including ALL subdirectories /// +/// Use responsibly. +/// /// ## Async /// /// For the `async` version, see: [`crate::list_nested_directories`] @@ -356,6 +364,8 @@ pub fn list_directories_with_filter>( /// Lists directories in a given directory (including ALL subdirectories) matching a filter pattern. /// +/// Use responsibly. +/// /// This pattern can be a `String`, `PathBuf`, or a [`regex::Regex`] pattern. /// /// ## Async From 154bd0dc0ee45e2a61e7c77960bc3bdad93250cb Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 18:04:25 +0000 Subject: [PATCH 096/105] doc: README update --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b0b2f16..9da80ef 100644 --- a/README.md +++ b/README.md @@ -20,5 +20,24 @@ Add to your `Cargo.toml` ```toml [dependencies] -filetools = "0.2.0" +filetools = "0.3.0" +``` + +Then import into your project: + +```rust +use filetools::{FtFilter, list_nested_files_with_filter}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Get all Lua files in the Neovim directory + let root_path = "/home/user/.config/nvim"; + let filter = FtFilter::Raw("lua".to_string()); + let lua_files = list_nested_files_with_filter(&root_path, filter).await?; + + // Delete them all, we hate Lua + for lua_file in lua_files.into_iter() { + tokio::fs::remove_file(lua_file).await?; + } +} ``` From 3b6576020cf9e8cba42cc08173d659a65e050d00 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 18:12:10 +0000 Subject: [PATCH 097/105] chore: added LICENSEs and update Cargo TOML --- Cargo.toml | 2 +- LICENSE-APACHE | 202 +++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE-MIT | 22 ++++++ 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT diff --git a/Cargo.toml b/Cargo.toml index 934ce30..d70c30c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" description = "General filehelpers to make some file operations easier" repository = "https://github.com/Tyrannican/filetools-rs" license = "MIT OR Apache-2.0" -keywords = ["files", "utility", "filesystem"] +keywords = ["files", "utility", "filesystem", "path"] categories = ["filesystem"] readme = "README.md" diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..1e5006d --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..632d183 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2024 Graham Keenan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + From d3a73631e3b29f3e1716975cb6db02aaec9c05d9 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 18:15:07 +0000 Subject: [PATCH 098/105] docs: minor --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d70c30c..2ab5be2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "filetools" version = "0.3.0" authors = ["Graham Keenan "] edition = "2021" -description = "General filehelpers to make some file operations easier" +description = "Helper functions for path operations" repository = "https://github.com/Tyrannican/filetools-rs" license = "MIT OR Apache-2.0" keywords = ["files", "utility", "filesystem", "path"] From b7f27109fd46038203fca1d3c62d953dfda5cc4d Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 20:01:37 +0000 Subject: [PATCH 099/105] docs: added example to lib.rs --- README.md | 2 ++ src/lib.rs | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/README.md b/README.md index 9da80ef..371161c 100644 --- a/README.md +++ b/README.md @@ -39,5 +39,7 @@ async fn main() -> anyhow::Result<()> { for lua_file in lua_files.into_iter() { tokio::fs::remove_file(lua_file).await?; } + + Ok(()) } ``` diff --git a/src/lib.rs b/src/lib.rs index f40db6e..7b03bcd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,27 @@ //! //! The operations in this crate are designed for async/await, however sync variations //! of the operations exist in the [`crate::sync`] module. +//! +//! # Example +//! +//! ```rust,no_run +//! use filetools::{FtFilter, list_nested_files_with_filter}; +//! +//! #[tokio::main] +//! async fn main() -> anyhow::Result<()> { +//! // Get all Lua files in the Neovim directory +//! let root_path = "/home/user/.config/nvim"; +//! let filter = FtFilter::Raw("lua".to_string()); +//! let lua_files = list_nested_files_with_filter(&root_path, filter).await?; +//! +//! // Delete them all, we hate Lua +//! for lua_file in lua_files.into_iter() { +//! tokio::fs::remove_file(lua_file).await?; +//! } +//! +//! Ok(()) +//! } +//! ``` pub mod naming; pub mod sync; From e18229645ef6a0419468f1a26a1f9745392c8714 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 20:13:13 +0000 Subject: [PATCH 100/105] chore: Clippy fixes --- src/lib.rs | 27 ++++++++++++--------------- src/naming.rs | 24 +++++++++++------------- src/sync.rs | 16 ++++++++-------- src/util.rs | 24 ++++++++++++------------ 4 files changed, 43 insertions(+), 48 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7b03bcd..9cf23dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,13 +91,10 @@ pub enum FtFilter { /// ``` pub fn is_subdir(path: impl AsRef, dir: impl AsRef) -> bool { for component in path.as_ref().components() { - match component { - Component::Normal(p) => { - if p == dir.as_ref().as_os_str() { - return true; - } + if let Component::Normal(p) = component { + if p == dir.as_ref().as_os_str() { + return true; } - _ => {} } } @@ -122,7 +119,7 @@ pub fn is_subdir(path: impl AsRef, dir: impl AsRef) -> bool { pub fn path_contains(path: impl AsRef, pattern: impl AsRef /* maybe */) -> bool { if let Some(p) = path.as_ref().to_str() { if let Some(pat) = pattern.as_ref().to_str() { - return p.contains(&pat); + return p.contains(pat); } } @@ -275,7 +272,7 @@ pub async fn list_files + Send>(path: P) -> Result> "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::FileNoRec, None).await + iteritems(path, FtIterItemState::File, None).await } /// Lists all files in a directory including ALL subdirectories @@ -315,7 +312,7 @@ pub async fn list_nested_files + Send>(path: P) -> Result + Send>( "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::FileNoRec, Some(&pattern)).await + iteritems(path, FtIterItemState::File, Some(&pattern)).await } /// Lists files in a folder (including ALL subdirectories) matching a filter pattern. @@ -427,7 +424,7 @@ pub async fn list_nested_files_with_filter + Send>( "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::FileRec, Some(&pattern)).await + iteritems(path, FtIterItemState::RFile, Some(&pattern)).await } /// Lists all directories in the given directory (not including subdirectories). @@ -464,7 +461,7 @@ pub async fn list_directories + Send>(path: P) -> Result + Send>(path: P) -> Result + Send>(path: P) -> Result> { anyhow::ensure!(path.as_ref().exists(), "path does not exist"); - iteritems(path, FtIterItemState::DirRec, None).await + iteritems(path, FtIterItemState::RDir, None).await } /// Lists directories in a given directory (not including subdirectories) matching a filter pattern. @@ -554,7 +551,7 @@ pub async fn list_directories_with_filter + Send>( "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::DirNoRec, Some(&filter)).await + iteritems(path, FtIterItemState::Dir, Some(&filter)).await } /// Lists directories in a given directory (including ALL subdirectories) matching a filter pattern. @@ -611,7 +608,7 @@ pub async fn list_nested_directories_with_filter + Send>( "path should be a directory, not a file" ); - iteritems(path, FtIterItemState::DirRec, Some(&filter)).await + iteritems(path, FtIterItemState::RDir, Some(&filter)).await } #[cfg(test)] diff --git a/src/naming.rs b/src/naming.rs index 673ddbc..146b75e 100644 --- a/src/naming.rs +++ b/src/naming.rs @@ -6,21 +6,19 @@ //! use std::path::PathBuf; //! use filetools::naming; //! -//! fn main() { -//! // Generates the name `test.pdf` -//! let custom_name = naming::generate_name("test", "pdf"); +//! // Generates the name `test.pdf` +//! let custom_name = naming::generate_name("test", "pdf"); //! -//! // Name will be suffixed by the current time it was generated -//! // E.g. `test_[Timestamp].pdf` -//! let timestamped_name = naming::generate_timestamped_name("test", "pdf"); +//! // Name will be suffixed by the current time it was generated +//! // E.g. `test_[Timestamp].pdf` +//! let timestamped_name = naming::generate_timestamped_name("test", "pdf"); //! -//! // Random name is a UUIDv4 string suffixed by the extension -//! // E.g. `00762527-012a-43c1-a673-cad9bc5eef64.pdf` -//! let random_name = naming::generate_uuid4_name("pdf"); +//! // Random name is a UUIDv4 string suffixed by the extension +//! // E.g. `00762527-012a-43c1-a673-cad9bc5eef64.pdf` +//! let random_name = naming::generate_uuid4_name("pdf"); //! -//! // N-digit name is a number prefixed by X zeros (e.g. 0005.pdf) -//! let n_digit_name = naming::generate_n_digit_name(5, 4, "pdf"); -//! } +//! // N-digit name is a number prefixed by X zeros (e.g. 0005.pdf) +//! let n_digit_name = naming::generate_n_digit_name(5, 4, "pdf"); //! ``` //! @@ -90,7 +88,7 @@ pub fn generate_timestamped_name(fname: &str, ext: &str) -> PathBuf { pub fn generate_uuid4_name(ext: &str) -> PathBuf { let unique = Uuid::new_v4(); - PathBuf::from(format!("{}{}", unique.to_string(), make_extension(ext))) + PathBuf::from(format!("{}{}", unique, make_extension(ext))) } /// Generates a `PathBuf` from a `number` prefixed by `n_digits` zeros. diff --git a/src/sync.rs b/src/sync.rs index 87e4ca4..3934759 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -129,7 +129,7 @@ pub fn list_files>(path: P) -> Result>> { "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::FileNoRec, None) + iteritems_sync(path, FtIterItemState::File, None) } /// Lists all files in a directory including ALL subdirectories @@ -164,7 +164,7 @@ pub fn list_nested_files>(path: P) -> Result> { "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::FileRec, None) + iteritems_sync(path, FtIterItemState::RFile, None) } /// Lists files in a folder (not including subdirectories) matching a filter pattern. @@ -206,7 +206,7 @@ pub fn list_files_with_filter>(path: P, filter: FtFilter) -> Resu "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::FileNoRec, Some(&filter)) + iteritems_sync(path, FtIterItemState::File, Some(&filter)) } /// Lists files in a folder (including ALL subdirectories) matching a filter pattern. @@ -251,7 +251,7 @@ pub fn list_nested_files_with_filter>( "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::FileRec, Some(&filter)) + iteritems_sync(path, FtIterItemState::RFile, Some(&filter)) } /// Lists all directories in the given directory (not including subdirectories). @@ -283,7 +283,7 @@ pub fn list_directories>(path: P) -> Result> path.as_ref().is_dir(), "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::DirNoRec, None) + iteritems_sync(path, FtIterItemState::Dir, None) } /// Lists all directories in a directory including ALL subdirectories @@ -317,7 +317,7 @@ pub fn list_nested_directories + Send>(path: P) -> Result>( path.as_ref().is_dir(), "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::DirNoRec, Some(&filter)) + iteritems_sync(path, FtIterItemState::Dir, Some(&filter)) } /// Lists directories in a given directory (including ALL subdirectories) matching a filter pattern. @@ -403,7 +403,7 @@ pub fn list_nested_directories_with_filter>( path.as_ref().is_dir(), "path should be a directory, not a file" ); - iteritems_sync(path, FtIterItemState::DirRec, Some(&filter)) + iteritems_sync(path, FtIterItemState::RDir, Some(&filter)) } // No tests needed cause these are tested in the main crate diff --git a/src/util.rs b/src/util.rs index 363d0f0..2823a74 100644 --- a/src/util.rs +++ b/src/util.rs @@ -12,16 +12,16 @@ use tokio::fs; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum FtIterItemState { /// Iterate files with no recursion - FileNoRec, + File, /// Iterate files with recursion - FileRec, + RFile, /// Iterate directories with no recursion - DirNoRec, + Dir, /// Iterate directories with recursion - DirRec, + RDir, } /// Helper function to determine if an path item is valid based on the supplied filter @@ -75,24 +75,24 @@ pub(crate) async fn iteritems + Send>( }; match iterstate { - FtIterItemState::FileNoRec => { + FtIterItemState::File => { if e_path.is_file() && filter_pass { items.push(e_path); } } - FtIterItemState::FileRec => { + FtIterItemState::RFile => { if e_path.is_file() && filter_pass { items.push(e_path) } else if e_path.is_dir() { items.extend(iteritems(e_path, iterstate, filter).await?); } } - FtIterItemState::DirNoRec => { + FtIterItemState::Dir => { if e_path.is_dir() && filter_pass { items.push(e_path); } } - FtIterItemState::DirRec => { + FtIterItemState::RDir => { if e_path.is_dir() { if filter_pass { items.push(e_path.clone()); @@ -126,24 +126,24 @@ pub(crate) fn iteritems_sync>( None => true, }; match iterstate { - FtIterItemState::FileNoRec => { + FtIterItemState::File => { if e_path.is_file() && filter_pass { items.push(e_path); } } - FtIterItemState::FileRec => { + FtIterItemState::RFile => { if e_path.is_file() && filter_pass { items.push(e_path) } else if e_path.is_dir() { items.extend(iteritems_sync(e_path, iterstate, filter)?); } } - FtIterItemState::DirNoRec => { + FtIterItemState::Dir => { if e_path.is_dir() && filter_pass { items.push(e_path); } } - FtIterItemState::DirRec => { + FtIterItemState::RDir => { if e_path.is_dir() { if filter_pass { items.push(e_path.clone()); From f50a66f526e0bafde33a865dedf40b8ee15b349a Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 20:14:24 +0000 Subject: [PATCH 101/105] doc: minor doc update --- src/util.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/util.rs b/src/util.rs index 2823a74..a0bd416 100644 --- a/src/util.rs +++ b/src/util.rs @@ -7,8 +7,6 @@ use std::path::{Path, PathBuf}; use tokio::fs; /// Determines the type of iteration performed by the `list_directories` and `list_files` functions -/// If the NoRec variation is used, only the current directory is considered -/// If the Rec variation is used, then ALL subdirectores are traversed #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum FtIterItemState { /// Iterate files with no recursion From fde8ecf746a4b9307018f7e58084a6dee5983156 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 20:18:54 +0000 Subject: [PATCH 102/105] removed unused workflows --- .github/workflows/nostd.yml | 30 ------------- .github/workflows/safety.yml | 84 ------------------------------------ 2 files changed, 114 deletions(-) delete mode 100644 .github/workflows/nostd.yml delete mode 100644 .github/workflows/safety.yml diff --git a/.github/workflows/nostd.yml b/.github/workflows/nostd.yml deleted file mode 100644 index c12227a..0000000 --- a/.github/workflows/nostd.yml +++ /dev/null @@ -1,30 +0,0 @@ -# This workflow checks whether the library is able to run without the std library (e.g., embedded). -# This entire file should be removed if this crate does not support no-std. See check.yml for -# information about how the concurrency cancellation and workflow triggering works -permissions: - contents: read -on: - push: - branches: [main] - pull_request: -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true -name: no-std -jobs: - nostd: - runs-on: ubuntu-latest - name: ${{ matrix.target }} - strategy: - matrix: - target: [thumbv7m-none-eabi, aarch64-unknown-none] - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install stable - uses: dtolnay/rust-toolchain@stable - - name: rustup target add ${{ matrix.target }} - run: rustup target add ${{ matrix.target }} - - name: cargo check - run: cargo check --target ${{ matrix.target }} --no-default-features diff --git a/.github/workflows/safety.yml b/.github/workflows/safety.yml deleted file mode 100644 index 6bdd055..0000000 --- a/.github/workflows/safety.yml +++ /dev/null @@ -1,84 +0,0 @@ -# This workflow runs checks for unsafe code. In crates that don't have any unsafe code, this can be -# removed. Runs: -# - miri - detects undefined behavior and memory leaks -# - address sanitizer - detects memory errors -# - leak sanitizer - detects memory leaks -# - loom - Permutation testing for concurrent code https://crates.io/crates/loom -# See check.yml for information about how the concurrency cancellation and workflow triggering works -permissions: - contents: read -on: - push: - branches: [main] - pull_request: -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true -name: safety -jobs: - sanitizers: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install nightly - uses: dtolnay/rust-toolchain@nightly - - run: | - # to get the symbolizer for debug symbol resolution - sudo apt install llvm - # to fix buggy leak analyzer: - # https://github.com/japaric/rust-san#unrealiable-leaksanitizer - # ensure there's a profile.dev section - if ! grep -qE '^[ \t]*[profile.dev]' Cargo.toml; then - echo >> Cargo.toml - echo '[profile.dev]' >> Cargo.toml - fi - # remove pre-existing opt-levels in profile.dev - sed -i '/^\s*\[profile.dev\]/,/^\s*\[/ {/^\s*opt-level/d}' Cargo.toml - # now set opt-level to 1 - sed -i '/^\s*\[profile.dev\]/a opt-level = 1' Cargo.toml - cat Cargo.toml - name: Enable debug symbols - - name: cargo test -Zsanitizer=address - # only --lib --tests b/c of https://github.com/rust-lang/rust/issues/53945 - run: cargo test --lib --tests --all-features --target x86_64-unknown-linux-gnu - env: - ASAN_OPTIONS: "detect_odr_violation=0:detect_leaks=0" - RUSTFLAGS: "-Z sanitizer=address" - - name: cargo test -Zsanitizer=leak - if: always() - run: cargo test --all-features --target x86_64-unknown-linux-gnu - env: - LSAN_OPTIONS: "suppressions=lsan-suppressions.txt" - RUSTFLAGS: "-Z sanitizer=leak" - miri: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - run: | - echo "NIGHTLY=nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/miri)" >> $GITHUB_ENV - - name: Install ${{ env.NIGHTLY }} - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ env.NIGHTLY }} - components: miri - - name: cargo miri test - run: cargo miri test - env: - MIRIFLAGS: "" - loom: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install stable - uses: dtolnay/rust-toolchain@stable - - name: cargo test --test loom - run: cargo test --release --test loom - env: - LOOM_MAX_PREEMPTIONS: 2 - RUSTFLAGS: "--cfg loom" From f438e64f52e816d75131ae65f3541dfa74fb7392 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 20:23:03 +0000 Subject: [PATCH 103/105] fix(ci): removed upload to codecov --- .github/workflows/test.yml | 47 -------------------------------------- Cargo.lock | 3 ++- 2 files changed, 2 insertions(+), 48 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f7540ae..470ef8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,50 +107,3 @@ jobs: run: cargo generate-lockfile - name: cargo test run: cargo test --locked --all-features --all-targets - coverage: - # use llvm-cov to build and collect coverage and outputs in a format that - # is compatible with codecov.io - # - # note that codecov as of v4 requires that CODECOV_TOKEN from - # - # https://app.codecov.io/gh///settings - # - # is set in two places on your repo: - # - # - https://github.com/jonhoo/guardian/settings/secrets/actions - # - https://github.com/jonhoo/guardian/settings/secrets/dependabot - # - # (the former is needed for codecov uploads to work with Dependabot PRs) - # - # PRs coming from forks of your repo will not have access to the token, but - # for those, codecov allows uploading coverage reports without a token. - # it's all a little weird and inconvenient. see - # - # https://github.com/codecov/feedback/issues/112 - # - # for lots of more discussion - runs-on: ubuntu-latest - name: ubuntu / stable / coverage - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install stable - uses: dtolnay/rust-toolchain@stable - with: - components: llvm-tools-preview - - name: cargo install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - name: cargo generate-lockfile - if: hashFiles('Cargo.lock') == '' - run: cargo generate-lockfile - - name: cargo llvm-cov - run: cargo llvm-cov --locked --all-features --lcov --output-path lcov.info - - name: Record Rust version - run: echo "RUST=$(rustc --version)" >> "$GITHUB_ENV" - - name: Upload to codecov.io - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - env_vars: OS,RUST diff --git a/Cargo.lock b/Cargo.lock index b6841db..cbad189 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,7 +11,8 @@ dependencies = [ "gimli", ] -[[package]] +This library allows for simple filename generation and finding all files/folders in a directory. +[package]] name = "adler" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" From f72224c33e013e2e579120be9ee29cbe72d3659a Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 20:24:48 +0000 Subject: [PATCH 104/105] fix: wtf cargo lock --- Cargo.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbad189..b6841db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,8 +11,7 @@ dependencies = [ "gimli", ] -This library allows for simple filename generation and finding all files/folders in a directory. -[package]] +[[package]] name = "adler" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" From 24433cb63bd10bb257f37b399c6005a9ff1be094 Mon Sep 17 00:00:00 2001 From: Graham Keenan Date: Sat, 10 Feb 2024 20:29:59 +0000 Subject: [PATCH 105/105] chore: remove msrv from workflow, dont care rn --- .github/workflows/check.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c869ee7..aed5529 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -92,22 +92,3 @@ jobs: # --feature-powerset runs for every combination of features - name: cargo hack run: cargo hack --feature-powerset check - msrv: - # check that we can build using the minimal rust version that is specified by this crate - runs-on: ubuntu-latest - # we use a matrix here just because env can't be used in job names - # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability - strategy: - matrix: - msrv: ["1.56.1"] # 2021 edition requires 1.56 - name: ubuntu / ${{ matrix.msrv }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install ${{ matrix.msrv }} - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.msrv }} - - name: cargo +${{ matrix.msrv }} check - run: cargo check