Skip to content

Commit

Permalink
Parsers::NPM: handle dependencies on local paths in lockfiles (librar…
Browse files Browse the repository at this point in the history 
…iesio#592)

* Parsers::NPM: add local dependency output

* 8.8.0
  • Loading branch information
@mpace965 @andrew
mpace965 authored and andrew committed Jun 4, 2024
1 parent 6064f0d commit ae7e27d
Show file tree
Hide file tree
Showing 8 changed files with 278 additions and 38 deletions.
14 changes: 12 additions & 2 deletions lib/bibliothecary/parsers/npm.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,18 @@ def self.parse_package_lock_v2(manifest)
# "packages" is a flat object where each key is the installed location of the dep, e.g. node_modules/foo/node_modules/bar.
manifest
.fetch("packages")
.reject { |name, _dep| name == "" } # this is the lockfile's package itself
# there are a couple of scenarios where a package's name won't start with node_modules
# 1. name == "", this is the lockfile's package itself
# 2. when a package is a local path dependency, it will appear in package-lock.json twice.
# * One occurrence has the node_modules/ prefix in the name (which we keep)
# * The other occurrence's name is the path to the local dependency (which has less information, and is duplicative, so we discard)
.select { |name, _dep| name.start_with?("node_modules") }
.map do |name, dep|
{
name: name.split("node_modules/").last,
requirement: dep["version"],
requirement: dep["version"] || "*",
type: dep.fetch("dev", false) || dep.fetch("devOptional", false) ? "development" : "runtime",
local: dep.fetch("link", false),
}
end
end
Expand Down Expand Up @@ -106,6 +112,9 @@ def self.parse_manifest(file_contents, options: {}) # rubocop:disable Lint/Unuse
map_dependencies(manifest, "devDependencies", "development")
)
.reject { |dep| dep[:name].start_with?("//") } # Omit comment keys. They are valid in package.json: https://groups.google.com/g/nodejs/c/NmL7jdeuw0M/m/yTqI05DRQrIJ
.each do |dep|
dep[:local] = dep[:requirement].start_with?("file:")
end
end

def self.parse_yarn_lock(file_contents, options: {})
Expand All @@ -120,6 +129,7 @@ def self.parse_yarn_lock(file_contents, options: {})
requirement: dep[:version],
lockfile_requirement: dep[:requirement],
type: dep[:type],
local: dep[:requirement]&.start_with?("file:"),
}
end
end
Expand Down
2 changes: 1 addition & 1 deletion lib/bibliothecary/version.rb
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
module Bibliothecary
VERSION = "8.7.7"
VERSION = "8.8.0"
end
68 changes: 68 additions & 0 deletions spec/fixtures/npm-local-file/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 16 additions & 0 deletions spec/fixtures/npm-local-file/package.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"name": "npm-bad",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"left-pad": "^1.3.0",
"other-package": "file:src/other-package",
"react": "^18.3.1"
}
}
30 changes: 30 additions & 0 deletions spec/fixtures/npm-local-file/yarn.lock
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1


"js-tokens@^3.0.0 || ^4.0.0":
version "4.0.0"
resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz"
integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==

left-pad@^1.3.0:
version "1.3.0"
resolved "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"
integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==

loose-envify@^1.1.0:
version "1.4.0"
resolved "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz"
integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==
dependencies:
js-tokens "^3.0.0 || ^4.0.0"

"other-package@file:src/other-package":
version "1.0.0"

react@^18.3.1:
version "18.3.1"
resolved "https://registry.npmjs.org/react/-/react-18.3.1.tgz"
integrity sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==
dependencies:
loose-envify "^1.1.0"
2 changes: 1 addition & 1 deletion spec/fixtures/npm-lockfile-version-1/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

117 changes: 83 additions & 34 deletions spec/parsers/npm_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@
platform: "npm",
path: "package.json",
dependencies: [
{ name: "babel", requirement: "^4.6.6", type: "runtime" },
{ name: "mocha", requirement: "^2.2.1", type: "development" },
{ name: "babel", requirement: "^4.6.6", type: "runtime", local: false },
{ name: "mocha", requirement: "^2.2.1", type: "development", local: false },
],
kind: "manifest",
success: true,
Expand Down Expand Up @@ -115,26 +115,26 @@
platform: "npm",
path: "yarn.lock",
dependencies: [
{ name: "body-parser", lockfile_requirement: "^1.15.2", requirement: "1.16.1", type: "runtime" },
{ name: "bytes", lockfile_requirement: "2.4.0", requirement: "2.4.0", type: "runtime" },
{ name: "content-type", lockfile_requirement: "~1.0.2", requirement: "1.0.2", type: "runtime" },
{ name: "debug", lockfile_requirement: "2.6.1", requirement: "2.6.1", type: "runtime" },
{ name: "depd", lockfile_requirement: "~1.1.0", requirement: "1.1.0", type: "runtime" },
{ name: "ee-first", lockfile_requirement: "1.1.1", requirement: "1.1.1", type: "runtime" },
{ name: "http-errors", lockfile_requirement: "~1.5.1", requirement: "1.5.1", type: "runtime" },
{ name: "iconv-lite", lockfile_requirement: "0.4.15", requirement: "0.4.15", type: "runtime" },
{ name: "inherits", lockfile_requirement: "2.0.3", requirement: "2.0.3", type: "runtime" },
{ name: "media-typer", lockfile_requirement: "0.3.0", requirement: "0.3.0", type: "runtime" },
{ name: "mime-db", lockfile_requirement: "~1.26.0", requirement: "1.26.0", type: "runtime" },
{ name: "mime-types", lockfile_requirement: "~2.1.13", requirement: "2.1.14", type: "runtime" },
{ name: "ms", lockfile_requirement: "0.7.2", requirement: "0.7.2", type: "runtime" },
{ name: "on-finished", lockfile_requirement: "~2.3.0", requirement: "2.3.0", type: "runtime" },
{ name: "qs", lockfile_requirement: "6.2.1", requirement: "6.2.1", type: "runtime" },
{ name: "raw-body", lockfile_requirement: "~2.2.0", requirement: "2.2.0", type: "runtime" },
{ name: "setprototypeof", lockfile_requirement: "1.0.2", requirement: "1.0.2", type: "runtime" },
{ name: "statuses", lockfile_requirement: ">= 1.3.1 < 2", requirement: "1.3.1", type: "runtime" },
{ name: "type-is", lockfile_requirement: "~1.6.14", requirement: "1.6.14", type: "runtime" },
{ name: "unpipe", lockfile_requirement: "1.0.0", requirement: "1.0.0", type: "runtime" },
{ name: "body-parser", lockfile_requirement: "^1.15.2", requirement: "1.16.1", type: "runtime", local: false },
{ name: "bytes", lockfile_requirement: "2.4.0", requirement: "2.4.0", type: "runtime", local: false },
{ name: "content-type", lockfile_requirement: "~1.0.2", requirement: "1.0.2", type: "runtime", local: false },
{ name: "debug", lockfile_requirement: "2.6.1", requirement: "2.6.1", type: "runtime", local: false },
{ name: "depd", lockfile_requirement: "~1.1.0", requirement: "1.1.0", type: "runtime", local: false },
{ name: "ee-first", lockfile_requirement: "1.1.1", requirement: "1.1.1", type: "runtime", local: false },
{ name: "http-errors", lockfile_requirement: "~1.5.1", requirement: "1.5.1", type: "runtime", local: false },
{ name: "iconv-lite", lockfile_requirement: "0.4.15", requirement: "0.4.15", type: "runtime", local: false },
{ name: "inherits", lockfile_requirement: "2.0.3", requirement: "2.0.3", type: "runtime", local: false },
{ name: "media-typer", lockfile_requirement: "0.3.0", requirement: "0.3.0", type: "runtime", local: false },
{ name: "mime-db", lockfile_requirement: "~1.26.0", requirement: "1.26.0", type: "runtime", local: false },
{ name: "mime-types", lockfile_requirement: "~2.1.13", requirement: "2.1.14", type: "runtime", local: false },
{ name: "ms", lockfile_requirement: "0.7.2", requirement: "0.7.2", type: "runtime", local: false },
{ name: "on-finished", lockfile_requirement: "~2.3.0", requirement: "2.3.0", type: "runtime", local: false },
{ name: "qs", lockfile_requirement: "6.2.1", requirement: "6.2.1", type: "runtime", local: false },
{ name: "raw-body", lockfile_requirement: "~2.2.0", requirement: "2.2.0", type: "runtime", local: false },
{ name: "setprototypeof", lockfile_requirement: "1.0.2", requirement: "1.0.2", type: "runtime", local: false },
{ name: "statuses", lockfile_requirement: ">= 1.3.1 < 2", requirement: "1.3.1", type: "runtime", local: false },
{ name: "type-is", lockfile_requirement: "~1.6.14", requirement: "1.6.14", type: "runtime", local: false },
{ name: "unpipe", lockfile_requirement: "1.0.0", requirement: "1.0.0", type: "runtime", local: false },
],
kind: "lockfile",
success: true,
Expand All @@ -146,7 +146,7 @@
platform: "npm",
path: "yarn.lock",
dependencies: [
{ name: "vue", lockfile_requirement: "https://github.com/vuejs/vue.git#v2.6.12", requirement: "2.6.12", type: "runtime" },
{ name: "vue", lockfile_requirement: "https://github.com/vuejs/vue.git#v2.6.12", requirement: "2.6.12", type: "runtime", local: false },
],
kind: "lockfile",
success: true,
Expand All @@ -158,22 +158,22 @@
platform: "npm",
path: "package.json",
dependencies: [
{ name: "vue", requirement: "https://github.com/vuejs/vue.git#v2.6.12", type: "runtime" },
{ name: "vue", requirement: "https://github.com/vuejs/vue.git#v2.6.12", type: "runtime", local: false },
],
kind: "manifest",
success: true,
})
end

it "wont load package-lock.json from a package.json" do
expect(described_class.analyse_contents("package.json", load_fixture("package-lock.json"))).to eq({
expect(described_class.analyse_contents("package.json", load_fixture("package-lock.json"))).to match({
platform: "npm",
path: "package.json",
dependencies: nil,
kind: "manifest",
success: false,
error_message: "package.json: appears to be a lockfile rather than manifest format",
error_location: "parsers/npm.rb:102:in `parse_manifest'",
error_location: match("in `parse_manifest'"),
})
end

Expand All @@ -187,6 +187,55 @@
})
end

context "with local path dependencies" do
it "parses local path dependencies from package.json" do
expect(described_class.analyse_contents("package.json", load_fixture("npm-local-file/package.json"))).to eq({
platform: "npm",
path: "package.json",
dependencies: [
{ name: "left-pad", requirement: "^1.3.0", type: "runtime", local: false },
{ name: "other-package", requirement: "file:src/other-package", type: "runtime", local: true },
{ name: "react", requirement: "^18.3.1", type: "runtime", local: false },
],
kind: "manifest",
success: true,
})
end

it "parses local path dependencies from package-lock.json" do
expect(described_class.analyse_contents("package-lock.json", load_fixture("npm-local-file/package-lock.json"))).to eq({
platform: "npm",
path: "package-lock.json",
dependencies: [
{ name: "js-tokens", requirement: "4.0.0", type: "runtime", local: false },
{ name: "left-pad", requirement: "1.3.0", type: "runtime", local: false },
{ name: "lodash", requirement: "4.17.21", type: "development", local: false },
{ name: "loose-envify", requirement: "1.4.0", type: "runtime", local: false },
{ name: "other-package", requirement: "*", type: "runtime", local: true },
{ name: "react", requirement: "18.3.1", type: "runtime", local: false },
],
kind: "lockfile",
success: true,
})
end

it "parses local path dependencies from yarn.lock", :vcr do
expect(described_class.analyse_contents("yarn.lock", load_fixture("npm-local-file/yarn.lock"))).to eq({
platform: "npm",
path: "yarn.lock",
dependencies: [
{ name: "js-tokens", requirement: "4.0.0", lockfile_requirement: "^3.0.0 || ^4.0.0", type: "runtime", local: false },
{ name: "left-pad", requirement: "1.3.0", lockfile_requirement: "^1.3.0", type: "runtime", local: false },
{ name: "loose-envify", requirement: "1.4.0", lockfile_requirement: "^1.1.0", type: "runtime", local: false },
{ name: "other-package", requirement: "1.0.0", lockfile_requirement: "file:src/other-package", type: "runtime", local: true },
{ name: "react", requirement: "18.3.1", lockfile_requirement: "^18.3.1", type: "runtime", local: false },
],
kind: "lockfile",
success: true,
})
end
end

it "parses package-lock.json with scm based versions" do
contents = JSON.dump(
{
Expand Down Expand Up @@ -264,9 +313,9 @@
it "parses dependencies that have multiple versions in package-lock.json" do
expect(described_class.analyse_contents("package-lock.json", load_fixture("multiple_versions/package-lock.json"))).to eq({
dependencies: [
{ name: "find-versions", requirement: "4.0.0", type: "runtime" },
{ name: "semver-regex", requirement: "3.1.3", type: "runtime" },
{ name: "semver-regex", requirement: "4.0.2", type: "runtime" },
{ name: "find-versions", requirement: "4.0.0", type: "runtime", local: false },
{ name: "semver-regex", requirement: "3.1.3", type: "runtime", local: false },
{ name: "semver-regex", requirement: "4.0.2", type: "runtime", local: false },
],
kind: "lockfile",
path: "package-lock.json",
Expand All @@ -278,9 +327,9 @@
it "parses dependencies that have multiple versions in yarn.json", :vcr do
expect(described_class.analyse_contents("yarn.lock", load_fixture("multiple_versions/yarn.lock"))).to eq({
dependencies: [
{ lockfile_requirement: "4.0.0", name: "find-versions", requirement: "4.0.0", type: "runtime" },
{ lockfile_requirement: "^3.1.2", name: "semver-regex", requirement: "3.1.3", type: "runtime" },
{ lockfile_requirement: "^4.0.0", name: "semver-regex", requirement: "4.0.2", type: "runtime" },
{ lockfile_requirement: "4.0.0", name: "find-versions", requirement: "4.0.0", type: "runtime", local: false },
{ lockfile_requirement: "^3.1.2", name: "semver-regex", requirement: "3.1.3", type: "runtime", local: false },
{ lockfile_requirement: "^4.0.0", name: "semver-regex", requirement: "4.0.2", type: "runtime", local: false },
],
kind: "lockfile",
path: "yarn.lock",
Expand Down Expand Up @@ -324,7 +373,7 @@
expect(analysis).to eq({
platform: "npm",
path: "npm-lockfile-version-2/package-lock.json",
dependencies: [{ name: "find-versions", requirement: "4.0.0", type: "runtime" }, { name: "semver-regex", requirement: "3.1.4", type: "runtime" }, { name: "semver-regex", requirement: "4.0.5", type: "runtime" }],
dependencies: [{ name: "find-versions", requirement: "4.0.0", type: "runtime", local: false }, { name: "semver-regex", requirement: "3.1.4", type: "runtime", local: false }, { name: "semver-regex", requirement: "4.0.5", type: "runtime", local: false }],
kind: "lockfile",
success: true,
})
Expand All @@ -335,7 +384,7 @@
expect(analysis).to eq({
platform: "npm",
path: "npm-lockfile-version-3/package-lock.json",
dependencies: [{ name: "find-versions", requirement: "4.0.0", type: "runtime" }, { name: "semver-regex", requirement: "3.1.4", type: "runtime" }, { name: "semver-regex", requirement: "4.0.5", type: "runtime" }],
dependencies: [{ name: "find-versions", requirement: "4.0.0", type: "runtime", local: false }, { name: "semver-regex", requirement: "3.1.4", type: "runtime", local: false }, { name: "semver-regex", requirement: "4.0.5", type: "runtime", local: false }],
kind: "lockfile",
success: true,
})
Expand Down
Loading

0 comments on commit ae7e27d

Please sign in to comment.