Skip to content

Commit

Permalink
fix: do not use excessive memory while parsing packfiles (#5)
Browse files Browse the repository at this point in the history
  • Loading branch information
ForbesLindesay authored Mar 6, 2022
1 parent 114b5d3 commit e7afeda
Show file tree
Hide file tree
Showing 18 changed files with 883 additions and 423 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
node_modules
13 changes: 13 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,16 @@ jobs:
- run: yarn install --frozen-lockfile
- run: yarn build
- run: yarn test

test-memory-usage:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
with:
node-version: 16.x
- run: yarn install --frozen-lockfile
- run: yarn build
- name: Check memory usage for packfile parser
run: node scripts/test-limited-memory-parse scripts/packfiles/nodejs-node.dat
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -96,4 +96,6 @@ bundle-cache
package-entry-points.json

secrets/
packages/packfile/samples/output.pack
packages/packfile/samples/output.pack

temp/
7 changes: 7 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM git-client-deps

WORKDIR /app

ADD . /app

CMD node --max_old_space_size=128 scripts/test-parse
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"is-builtin-module": "^3.0.0",
"jest": "^26.0.1",
"lint-staged": "^10.1.3",
"parameter-reducers": "^2.0.0",
"prettier": "^2.0.4",
"react": "^17.0.1",
"react-dom": "^17.0.1",
Expand Down
2 changes: 1 addition & 1 deletion packages/http/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ After the initial request, all other requests require an object detailing th ser

#### DEFAULT_HTTP_HANDLER

An implementation of `HttpInterface` using `cross-fetch`. You can wrap these if you need to set additional headers (e.g. for auth).
An implementation of `HttpInterface` using `http-basic`. You can wrap these if you need to set additional headers (e.g. for auth).

### Methods

Expand Down
1 change: 0 additions & 1 deletion packages/http/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
},
"dependencies": {
"@rollingversions/git-protocol": "^0.0.0",
"cross-fetch": "^3.0.6",
"http-basic": "^8.1.3",
"https-proxy-agent": "^5.0.0"
},
Expand Down
71 changes: 69 additions & 2 deletions packages/http/src/fetchObjects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ import {
parseFetchResponse,
FetchCommand,
FetchResponseEntryObject,
FetchCommandOutputOptions,
parseFetchResponseV2,
FetchCommandOutputOptionsV2,
} from '@rollingversions/git-protocol';
import {ContextWithServerCapabilities} from './Context';

Expand All @@ -16,7 +19,12 @@ export default async function fetchObjects<
>(
repoURL: URL,
command: FetchCommand,
ctx: ContextWithServerCapabilities<THeaders>,
{
raw,
store,
storeMode,
...ctx
}: ContextWithServerCapabilities<THeaders> & FetchCommandOutputOptions,
) {
const url = new URL(
`${
Expand Down Expand Up @@ -56,5 +64,64 @@ export default async function fetchObjects<
)}`,
);
}
return parseFetchResponse(response.body);
return parseFetchResponse(response.body, {raw, store, storeMode});
}

export async function* fetchObjectsV2<
THeaders extends {set(name: string, value: string): unknown}
>(
repoURL: URL,
command: FetchCommand,
{
onProgress,
store,
storeMode,
...ctx
}: ContextWithServerCapabilities<THeaders> & FetchCommandOutputOptionsV2,
) {
const url = new URL(
`${
repoURL.href.endsWith('.git') ? repoURL.href : `${repoURL.href}.git`
}/git-upload-pack`,
);
const headers = ctx.http.createHeaders(url);
headers.set('accept', 'application/x-git-upload-pack-result');
headers.set('content-type', 'application/x-git-upload-pack-request');
headers.set('git-protocol', 'version=2');
headers.set('user-agent', ctx.agent);

const response = await ctx.http.post(
url,
headers,
composeFetchCommand(
command,
new Map(
[
['agent', ctx.agent] as const,
...defaultCapabilities,
].filter(([key]) => ctx.serverCapabilities.has(key)),
),
),
);
if (response.statusCode !== 200) {
const body = await new Promise<Buffer>((resolve, reject) => {
const body: Buffer[] = [];
response.body
.on(`data`, (chunk) => body.push(chunk))
.on(`error`, reject)
.on(`end`, () => resolve(Buffer.concat(body)));
});
throw new Error(
`Git server responded with status ${response.statusCode}: ${body.toString(
`utf8`,
)}`,
);
}
for await (const entry of parseFetchResponseV2(response.body, {
onProgress,
store,
storeMode,
})) {
yield entry;
}
}
4 changes: 1 addition & 3 deletions packages/packfile/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@
"build": "tsc"
},
"dependencies": {
"@rollingversions/git-core": "^0.0.0",
"@types/pako": "^1.0.2",
"pako": "^1.0.5"
"@rollingversions/git-core": "^0.0.0"
},
"devDependencies": {
"@types/node": "*"
Expand Down
Loading

0 comments on commit e7afeda

Please sign in to comment.