fix: remove es-git and use node.js streams (#2)

.gitignore

@@ -93,4 +93,7 @@ snowpack-cache
 app/tailwind.css.br
 
 bundle-cache
-package-entry-points.json
+package-entry-points.json
+
+secrets/
+packages/packfile/samples/output.pack

package.json

@@ -9,6 +9,7 @@
   "dependencies": {
     "@babel/runtime": "^7.12.5",
     "@forbeslindesay/tsconfig": "^2.0.0",
+    "@github-graph/api": "^2.2.1",
     "@rollup/plugin-commonjs": "^17.0.0",
     "@rollup/plugin-node-resolve": "^11.0.0",
     "@sucrase/jest-plugin": "^2.0.0",
@@ -17,6 +18,7 @@
     "@types/react-dom": "^17.0.0",
     "copy-dir": "^1.3.0",
     "husky": "^4.2.5",
+    "interrogator": "^1.1.0",
     "is-builtin-module": "^3.0.0",
     "jest": "^26.0.1",
     "lint-staged": "^10.1.3",
@@ -34,6 +36,7 @@
   "scripts": {
     "http": "yarn workspace @rollingversions/git-http",
     "objects": "yarn workspace @rollingversions/git-objects",
+    "packfile": "yarn workspace @rollingversions/git-packfile",
     "protocol": "yarn workspace @rollingversions/git-protocol",
     "streams": "yarn workspace @rollingversions/git-streams",
     "build": "yarn build:links && yarn build:ts && yarn build:rollup",

packages/core/README.md

@@ -0,0 +1,3 @@
+# @rollingversions/git-core
+
+Forked from https://github.com/mariusGundersen/es-git

packages/core/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@rollingversions/git-core",
+  "version": "0.0.0",
+  "main": "dist/index.cjs",
+  "module": "dist/index.mjs",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.cjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist/",
+    "lib/"
+  ],
+  "scripts": {
+    "build": "tsc"
+  },
+  "dependencies": {},
+  "devDependencies": {},
+  "peerDependencies": {},
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "MIT"
+}

packages/core/rollup.toml

@@ -0,0 +1 @@
+exports = 'named'

packages/core/src/index.ts

@@ -0,0 +1,52 @@
+export {
+  decode,
+  encode,
+  concat,
+  fromDec,
+  fromHex,
+  fromOct,
+  fromDecChar,
+  fromHexChar,
+  toHexChar,
+  NEWLINE,
+} from './utils';
+
+export enum Mask {
+  mask = 0o100000,
+  blob = 0o140000,
+  file = 0o160000,
+}
+
+export enum Mode {
+  tree = 0o040000,
+  blob = 0o100644,
+  file = 0o100644,
+  exec = 0o100755,
+  sym = 0o120000,
+  commit = 0o160000,
+}
+
+export enum Type {
+  unknown = 'unknown',
+  commit = 'commit',
+  tree = 'tree',
+  blob = 'blob',
+  tag = 'tag',
+}
+
+export function isBlob(mode: number) {
+  return (mode & Mask.blob) === Mask.mask;
+}
+
+export function isFile(mode: number) {
+  return (mode & Mask.file) === Mask.mask;
+}
+
+export function toType(mode: number) {
+  if (mode === Mode.commit) return Type.commit;
+  if (mode === Mode.tree) return Type.tree;
+  if ((mode & Mask.blob) === Mask.mask) return Type.blob;
+  return Type.unknown;
+}
+
+export type Hash = string;

packages/core/src/utils.test.ts

@@ -0,0 +1,13 @@
+import {toHexChar} from './utils';
+
+for (const [num, hexChar] of [
+  [0, '0'],
+  [1, '1'],
+  [9, '9'],
+  [10, 'a'],
+  [15, 'f'],
+] as const) {
+  test(`toHexChar(${num}) => ${hexChar}`, () => {
+    expect(String.fromCharCode(toHexChar(num))).toBe(hexChar);
+  });
+}

packages/core/src/utils.ts

@@ -0,0 +1,67 @@
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+export const NEWLINE = '\n'.charCodeAt(0);
+
+export function encode(text: string) {
+  return encoder.encode(text);
+}
+
+export function decode(binary: Uint8Array, start = 0, end = binary.length) {
+  if (start !== 0 || end !== binary.length) {
+    return decoder.decode(binary.subarray(start, end));
+  } else {
+    return decoder.decode(binary);
+  }
+}
+
+export function concat(...arrays: (Uint8Array | number[])[]): Uint8Array {
+  const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const arr of arrays) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}
+
+export function fromHex(binary: Uint8Array) {
+  let size = 0;
+  for (let i = 0; i < 4; i++) {
+    size = (size << 4) | fromHexChar(binary[i]);
+  }
+  return size;
+}
+
+export function fromHexChar(val: number) {
+  return val < 0x57 ? val - 0x30 : val - 0x57;
+}
+
+export function fromDec(buffer: Uint8Array, start: number, end: number) {
+  let val = 0;
+  while (start < end) {
+    val = val * 10 + fromDecChar(buffer[start++]);
+  }
+  return val;
+}
+
+export function fromDecChar(val: number) {
+  return val - 0x30;
+}
+
+export function fromOct(
+  buffer: Uint8Array,
+  start: number,
+  end: number,
+): number {
+  let val = 0;
+  while (start < end) {
+    val = (val << 3) + fromDecChar(buffer[start++]);
+  }
+  return val;
+}
+
+export function toHexChar(val: number) {
+  return val < 10 ? val + 0x30 : val + 0x57;
+}

packages/core/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@forbeslindesay/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "module": "ES2020",
+    "incremental": true,
+    "composite": true,
+    "rootDir": "src",
+    "tsBuildInfoFile": "lib/.tsbuildinfo",
+    "lib": ["DOM", "ES2020"]
+  },
+  "include": ["src"],
+  "references": [],
+}

packages/http/package.json

@@ -20,8 +20,8 @@
   },
   "dependencies": {
     "@rollingversions/git-protocol": "^0.0.0",
-    "@rollingversions/git-streams": "^0.0.0",
     "cross-fetch": "^3.0.6",
+    "http-basic": "^8.1.3",
     "https-proxy-agent": "^5.0.0"
   },
   "devDependencies": {},

packages/http/src/HttpInterface.ts

@@ -1,11 +1,16 @@
+export interface HttpResponse {
+  url: URL;
+  statusCode: number;
+  body: NodeJS.ReadableStream;
+}
 export default interface HttpInterface<
   THeaders extends {set(name: string, value: string): unknown}
 > {
   createHeaders(url: URL): THeaders;
-  get(url: URL, headers: THeaders): AsyncIterableIterator<Uint8Array>;
+  get(url: URL, headers: THeaders): Promise<HttpResponse>;
   post(
     url: URL,
     headers: THeaders,
-    body: AsyncIterableIterator<Uint8Array>,
-  ): AsyncIterableIterator<Uint8Array>;
+    body: NodeJS.ReadableStream,
+  ): Promise<HttpResponse>;
 }

packages/http/src/createHttpHandler.ts

@@ -1,56 +1,47 @@
-import fetch, {Headers} from 'cross-fetch';
-import {
-  asyncIteratorToStream,
-  streamToAsyncIterator,
-} from '@rollingversions/git-streams';
-import HttpInterface from './HttpInterface';
+import request from 'http-basic';
+import HttpInterface, {HttpResponse} from './HttpInterface';
 
 const createHttpHandler = (options: {
   [key: string]: any;
-}): HttpInterface<Headers> => ({
-  createHeaders: () => new Headers(),
-  get: async function* (url, headers) {
-    const response = await fetch(url.href, {...options, headers});
-    if (!response.ok) {
-      throw new Error(
-        `Server responded to ${url.href} with status code ${
-          response.status
-        }: ${await response.text()}`,
+}): HttpInterface<Map<string, string>> => ({
+  createHeaders: () => new Map(),
+  get: async (url, headers) => {
+    return await new Promise<HttpResponse>((resolve, reject) => {
+      request(
+        `GET`,
+        url.href,
+        {...options, headers: Object.fromEntries(headers.entries())},
+        (err, res) => {
+          if (err) reject(err);
+          else
+            resolve({
+              statusCode: res!.statusCode,
+              url: new URL(res!.url),
+              body: res!.body,
+            });
+        },
       );
-    }
-    if (response.body) {
-      for await (const chunk of streamToAsyncIterator(response.body)) {
-        yield chunk;
-      }
-    } else {
-      yield new Uint8Array(await response.arrayBuffer());
-    }
-  },
-  post: async function* (url, headers, body) {
-    const response = await fetch(url.href, {
-      ...options,
-      method: 'POST',
-      headers,
-      // @ts-expect-error - on NodeJS this is a NodeJS stream, in the browser it is a browser stream
-      body: await asyncIteratorToStream(body),
     });
-    if (!response.ok) {
-      console.error(
-        `Server responded to ${url.href} with status code ${response.status}`,
-      );
-      throw new Error(
-        `Server responded to ${url.href} with status code ${
-          response.status
-        }: ${await response.text()}`,
+  },
+  post: async (url, headers, body) => {
+    return await new Promise<HttpResponse>((resolve, reject) => {
+      body.pipe(
+        request(
+          `POST`,
+          url.href,
+          {...options, headers: Object.fromEntries(headers.entries())},
+          (err, res) => {
+            if (err) reject(err);
+            else
+              resolve({
+                statusCode: res!.statusCode,
+                url: new URL(res!.url),
+                body: res!.body,
+              });
+          },
+        ) as NodeJS.WritableStream,
       );
-    }
-    if (response.body) {
-      for await (const chunk of streamToAsyncIterator(response.body)) {
-        yield chunk;
-      }
-    } else {
-      yield new Uint8Array(await response.arrayBuffer());
-    }
+    });
   },
 });