Skip to content

Commit

Permalink
[Feat] add benchmarking page. (#142)
Browse files Browse the repository at this point in the history
* feat: added benchmarking MVP
  • Loading branch information
a-ghorbani authored Dec 21, 2024
1 parent cbe06d6 commit f8dcd16
Show file tree
Hide file tree
Showing 62 changed files with 3,878 additions and 73 deletions.
7 changes: 7 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
APP_RELEASE_STORE_PASSWORD=
APP_RELEASE_KEY_PASSWORD=
FIREBASE_FUNCTIONS_URL=

# Debug tokens (only for local development)
APPCHECK_DEBUG_TOKEN_ANDROID=
APPCHECK_DEBUG_TOKEN_IOS=
14 changes: 13 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ jobs:
workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
create_credentials_file: true

# Add this step before the Android build
- name: Create .env file
run: |
echo "FIREBASE_FUNCTIONS_URL=${{ vars.FIREBASE_FUNCTIONS_URL }}" > .env
# Step 10: Build and upload Android app to Alpha track (includes building APK and Bundle)
- name: Build and upload Android app
working-directory: android
Expand All @@ -100,6 +105,7 @@ jobs:
GRADLE_USER_HOME: ${{ runner.temp }}/.gradle
# GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.auth.outputs.credentials_file_path }} # This is not supported by fastlane, we need to replace it with PLAY_STORE_JSON_KEY in the future.
PLAY_STORE_JSON_KEY: ${{ secrets.PLAY_STORE_SERVICE_ACCOUNT_JSON }}
GOOGLE_SERVICES_JSON: ${{ secrets.GOOGLE_SERVICES_JSON }}
run: |
echo "$PLAY_STORE_JSON_KEY" > play-store-key.json
bundle exec fastlane release_android_alpha
Expand Down Expand Up @@ -151,6 +157,11 @@ jobs:
working-directory: ios
run: pod install

# Add this step before the iOS build
- name: Create .env file
run: |
echo "FIREBASE_FUNCTIONS_URL=${{ vars.FIREBASE_FUNCTIONS_URL }}" > .env
# Step 6: Build and upload iOS app to TestFlight
- name: Build and upload iOS app
working-directory: ios
Expand All @@ -162,4 +173,5 @@ jobs:
APP_STORE_CONNECT_API_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_API_ISSUER_ID }}
APP_STORE_CONNECT_API_KEY_CONTENT: ${{ secrets.APP_STORE_CONNECT_API_KEY_CONTENT }}
APP_STORE_CONNECT_USER_ID: ${{ secrets.APP_STORE_CONNECT_USER_ID }}
run: bundle exec fastlane release_ios
GOOGLE_SERVICES_PLIST: ${{ secrets.GOOGLE_SERVICES_PLIST }}
run: bundle exec fastlane release_ios_testflight
9 changes: 9 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,12 @@ package-lock.json
# testing
/coverage
.env

# Firebase config files
android/app/google-services.json
ios/GoogleService-Info.plist

# Environment files
.env
.env.*
!.env.example
14 changes: 11 additions & 3 deletions App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,13 @@ import {

import {useTheme} from './src/hooks';
import {modelStore} from './src/store';
import {HeaderRight, SidebarContent} from './src/components';
import {ChatScreen, ModelsScreen, SettingsScreen} from './src/screens';
import {ModelsHeaderRight} from './src/components';
import {HeaderRight, SidebarContent, ModelsHeaderRight} from './src/components';
import {
ChatScreen,
ModelsScreen,
SettingsScreen,
BenchmarkScreen,
} from './src/screens';

const Drawer = createDrawerNavigator();

Expand Down Expand Up @@ -74,6 +78,10 @@ const App = observer(() => {
name="Settings"
component={gestureHandlerRootHOC(SettingsScreen)}
/>
<Drawer.Screen
name="Benchmark"
component={gestureHandlerRootHOC(BenchmarkScreen)}
/>
</Drawer.Navigator>
</NavigationContainer>
</PaperProvider>
Expand Down
11 changes: 11 additions & 0 deletions __mocks__/external/@react-native-firebase/app-check.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
const mockAppCheck = {
newReactNativeFirebaseAppCheckProvider: jest.fn().mockReturnValue({
configure: jest.fn(),
}),
initializeAppCheck: jest.fn(),
getToken: jest.fn().mockResolvedValue({token: 'mock-token'}),
};

export default () => ({
appCheck: () => mockAppCheck,
});
11 changes: 11 additions & 0 deletions __mocks__/external/@react-native-firebase/app.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
const mockFirebase = {
appCheck: jest.fn().mockReturnValue({
newReactNativeFirebaseAppCheckProvider: jest.fn().mockReturnValue({
configure: jest.fn(),
}),
initializeAppCheck: jest.fn(),
getToken: jest.fn().mockResolvedValue({token: 'mock-token'}),
}),
};

export default mockFirebase;
6 changes: 6 additions & 0 deletions __mocks__/external/react-native-device-info.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,11 @@ export default {
getUsedMemory: jest.fn(() => deviceInfo.usedMemory),
getVersion: jest.fn(() => deviceInfo.version),
getBuildNumber: jest.fn(() => deviceInfo.buildNumber),
isEmulator: jest.fn(() => false),
getBrand: jest.fn(() => 'Apple'),
getDevice: jest.fn(() => 'iPhone 12'),
getDeviceId: jest.fn(() => 'test-device-id'),
supportedAbis: jest.fn(() => ['arm64', 'arm64-v8a']),

// Not all methods are mocked, add any other methods from react-native-device-info that you use in your code
};
39 changes: 39 additions & 0 deletions __mocks__/stores/benchmarkStore.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import {BenchmarkResult} from '../../src/utils/types';
import {mockResult} from '../../jest/fixtures/benchmark';

// Create mock functions
const mockRemoveResult = jest.fn((timestamp: string) => {
benchmarkStore.results = benchmarkStore.results.filter(
result => result.timestamp !== timestamp,
);
});

const mockClearResults = jest.fn(() => {
benchmarkStore.results = [];
});

const mockAddResult = jest.fn((result: BenchmarkResult) => {
benchmarkStore.results.unshift(result);
});

const mockMarkAsSubmitted = jest.fn((uuid: string) => {
const result = benchmarkStore.results.find(r => r.uuid === uuid);
if (result) {
result.submitted = true;
}
});

const mockGetResultsByModel = jest.fn((modelId: string): BenchmarkResult[] => {
return benchmarkStore.results.filter(result => result.modelId === modelId);
});

// Define the mockBenchmarkStore
export const benchmarkStore = {
results: [mockResult],
addResult: mockAddResult,
removeResult: mockRemoveResult,
clearResults: mockClearResults,
markAsSubmitted: mockMarkAsSubmitted,
getResultsByModel: mockGetResultsByModel,
latestResult: mockResult,
};
4 changes: 4 additions & 0 deletions __mocks__/stores/modelStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ class MockModelStore {
get activeModel() {
return this.models.find(model => model.id === this.activeModelId);
}

get availableModels() {
return this.models.filter(model => model.isDownloaded);
}
}

export const mockModelStore = new MockModelStore();
4 changes: 4 additions & 0 deletions __mocks__/stores/uiStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ export class UIStore {
export const mockUiStore = {
colorScheme: 'light',
autoNavigatetoChat: false,
benchmarkShareDialog: {
shouldShow: true,
},
pageStates: {
modelsScreen: {
filters: [],
Expand All @@ -21,4 +24,5 @@ export const mockUiStore = {
setAutoNavigateToChat: jest.fn(),
setColorScheme: jest.fn(),
setDisplayMemUsage: jest.fn(),
setBenchmarkShareDialogPreference: jest.fn(),
};
9 changes: 9 additions & 0 deletions android/app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ apply plugin: "com.android.application"
apply plugin: "org.jetbrains.kotlin.android"
apply plugin: "com.facebook.react"

// Add the Google Services plugin dependency
// This is used exclusively for sending benchmarks (with user consent) to Hugging Face Spaces via Firebase.
// Firebase is used for App Check functionality, allowing unauthenticated users to submit their benchmark data securely.
apply plugin: "com.google.gms.google-services"

apply from: "../../node_modules/react-native-config/android/dotenv.gradle"

/**
Expand Down Expand Up @@ -128,6 +133,10 @@ dependencies {
} else {
implementation jscFlavor
}

implementation platform('com.google.firebase:firebase-bom:32.7.0')
implementation 'com.google.firebase:firebase-appcheck-playintegrity'
implementation 'com.google.firebase:firebase-appcheck-debug'
}

// apply from: file("../../node_modules/@react-native-community/cli-platform-android/native_modules.gradle"); applyNativeModulesAppBuildGradle(project)
Expand Down
87 changes: 87 additions & 0 deletions android/app/src/main/java/com/pocketpalai/DeviceInfoModule.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package com.pocketpal

import com.facebook.react.bridge.*
import android.os.Build
import java.io.File

class DeviceInfoModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) {

override fun getName(): String = "DeviceInfoModule"

@ReactMethod
fun getChipset(promise: Promise) {
try {
val chipset = Build.HARDWARE.takeUnless { it.isNullOrEmpty() } ?: Build.BOARD
promise.resolve(chipset)
} catch (e: Exception) {
promise.reject("ERROR", e.message)
}
}

@ReactMethod
fun getCPUInfo(promise: Promise) {
try {
val cpuInfo = Arguments.createMap()
cpuInfo.putInt("cores", Runtime.getRuntime().availableProcessors())

val processors = Arguments.createArray()
val features = mutableSetOf<String>()
val cpuInfoFile = File("/proc/cpuinfo")

if (cpuInfoFile.exists()) {
val cpuInfoLines = cpuInfoFile.readLines()
var currentProcessor = Arguments.createMap()
var hasData = false

for (line in cpuInfoLines) {
if (line.isEmpty() && hasData) {
processors.pushMap(currentProcessor)
currentProcessor = Arguments.createMap()
hasData = false
continue
}

val parts = line.split(":")
if (parts.size >= 2) {
val key = parts[0].trim()
val value = parts[1].trim()
when (key) {
"processor", "model name", "cpu MHz", "vendor_id" -> {
currentProcessor.putString(key, value)
hasData = true
}
"flags", "Features" -> { // "flags" for x86, "Features" for ARM
features.addAll(value.split(" ").filter { it.isNotEmpty() })
}
}
}
}

if (hasData) {
processors.pushMap(currentProcessor)
}

cpuInfo.putArray("processors", processors)

// Convert features set to array
val featuresArray = Arguments.createArray()
features.forEach { featuresArray.pushString(it) }
cpuInfo.putArray("features", featuresArray)

// ML-related CPU features detection
cpuInfo.putBoolean("hasFp16", features.any { it in setOf("fphp", "fp16") })
cpuInfo.putBoolean("hasDotProd", features.any { it in setOf("dotprod", "asimddp") })
cpuInfo.putBoolean("hasSve", features.any { it == "sve" })
cpuInfo.putBoolean("hasI8mm", features.any { it == "i8mm" })
}

if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
cpuInfo.putString("socModel", Build.SOC_MODEL)
}

promise.resolve(cpuInfo)
} catch (e: Exception) {
promise.reject("ERROR", e.message)
}
}
}
18 changes: 18 additions & 0 deletions android/app/src/main/java/com/pocketpalai/DeviceInfoPackage.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package com.pocketpal

import android.view.View
import com.facebook.react.ReactPackage
import com.facebook.react.bridge.NativeModule
import com.facebook.react.bridge.ReactApplicationContext
import com.facebook.react.uimanager.ReactShadowNode
import com.facebook.react.uimanager.ViewManager

class DeviceInfoPackage : ReactPackage {
override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
return listOf(DeviceInfoModule(reactContext))
}

override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<View, ReactShadowNode<*>>> {
return emptyList()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class MainApplication : Application(), ReactApplication {
PackageList(this).packages.apply {
// Packages that cannot be autolinked yet can be added manually here, for example:
// add(MyReactNativePackage())
add(DeviceInfoPackage())
}

override fun getJSMainModuleName(): String = "index"
Expand Down
5 changes: 5 additions & 0 deletions android/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ buildscript {
classpath("com.android.tools.build:gradle")
classpath("com.facebook.react:react-native-gradle-plugin")
classpath("org.jetbrains.kotlin:kotlin-gradle-plugin")

// Add the Google Services plugin dependency
// This is used exclusively for sending benchmarks (with user consent) to Hugging Face Spaces via Firebase.
// Firebase is used for App Check functionality, allowing unauthenticated users to submit their benchmark data securely.
classpath 'com.google.gms:google-services:4.4.2'
}
}

Expand Down
6 changes: 6 additions & 0 deletions android/fastlane/Fastfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ platform :android do
lane :release_android_alpha do
android_dir = File.expand_path('..', Dir.pwd)

# Create google-services.json from GitHub secret
File.write(
File.join(android_dir, "app/google-services.json"),
ENV["GOOGLE_SERVICES_JSON"]
)

# Build APK for GitHub Release
gradle(
task: "assemble",
Expand Down
5 changes: 4 additions & 1 deletion babel.config.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
module.exports = {
presets: ['module:@react-native/babel-preset'],
plugins: ['react-native-reanimated/plugin'],
plugins: [
'react-native-reanimated/plugin',
['module:react-native-dotenv', {moduleName: '@env'}],
],
};
5 changes: 5 additions & 0 deletions env.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
declare module '@env' {
export const FIREBASE_FUNCTIONS_URL: string;
export const APPCHECK_DEBUG_TOKEN_ANDROID: string;
export const APPCHECK_DEBUG_TOKEN_IOS: string;
}
Loading

0 comments on commit f8dcd16

Please sign in to comment.