Skip to content

Commit

Permalink
Merge pull request #35 from jasonjmcghee/fix-timeline-loading
Browse files Browse the repository at this point in the history 
careful profiling, fixing timeline loading, auto-pause recording on open timeline
  • Loading branch information
@jasonjmcghee
jasonjmcghee authored Dec 31, 2023
2 parents 950d887 + e2487bb commit 11b2288
Show file tree
Hide file tree
Showing 6 changed files with 145 additions and 91 deletions.
4 changes: 4 additions & 0 deletions rem.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
961C95F82B2E19B40093F228 /* remUITestsLaunchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95F72B2E19B40093F228 /* remUITestsLaunchTests.swift */; };
961C96132B2EB7DB0093F228 /* TimelineView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C96122B2EB7DB0093F228 /* TimelineView.swift */; };
961C96152B2EBEE50093F228 /* DB.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C96142B2EBEE50093F228 /* DB.swift */; };
9670E1352B41683B005728F5 /* ImageHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9670E1342B41683B005728F5 /* ImageHelper.swift */; };
969BA2EC2B3D1D46009EE9C6 /* SettingsManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */; };
969F3EFF2B3A8C4D0085787B /* HotKey in Frameworks */ = {isa = PBXBuildFile; productRef = 969F3EFE2B3A8C4D0085787B /* HotKey */; };
969F3F082B3B7C7C0085787B /* RemFileManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 969F3F072B3B7C7C0085787B /* RemFileManager.swift */; };
Expand Down Expand Up @@ -204,6 +205,7 @@
961C960D2B2E73840093F228 /* ffmpeg */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.executable"; path = ffmpeg; sourceTree = "<group>"; };
961C96122B2EB7DB0093F228 /* TimelineView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimelineView.swift; sourceTree = "<group>"; };
961C96142B2EBEE50093F228 /* DB.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DB.swift; sourceTree = "<group>"; };
9670E1342B41683B005728F5 /* ImageHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageHelper.swift; sourceTree = "<group>"; };
969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsManager.swift; sourceTree = "<group>"; };
969F3F072B3B7C7C0085787B /* RemFileManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemFileManager.swift; sourceTree = "<group>"; };
969F3F092B3B7F760085787B /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = "<group>"; };
Expand Down Expand Up @@ -298,6 +300,7 @@
969F3F0C2B3CCEC30085787B /* Ask.swift */,
969BA2EB2B3D1D46009EE9C6 /* SettingsManager.swift */,
96DBA3E62B403ED90000CFBE /* Timings.swift */,
9670E1342B41683B005728F5 /* ImageHelper.swift */,
);
path = rem;
sourceTree = "<group>";
Expand Down Expand Up @@ -664,6 +667,7 @@
969BA2EC2B3D1D46009EE9C6 /* SettingsManager.swift in Sources */,
969F3F0D2B3CCEC30085787B /* Ask.swift in Sources */,
96DBA3E72B403ED90000CFBE /* Timings.swift in Sources */,
9670E1352B41683B005728F5 /* ImageHelper.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
10 changes: 8 additions & 2 deletions rem/DB.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,17 @@ import AVFoundation
import Foundation
import SQLite
import Vision
import os

class DatabaseManager {
static let shared = DatabaseManager()
private var db: Connection
static var FPS: CMTimeScale = 25

private let logger = Logger(
subsystem: Bundle.main.bundleIdentifier!,
category: String(describing: DatabaseManager.self)
)
// Last 15 frames
let recentFramesThreshold = 15

Expand Down Expand Up @@ -132,6 +137,7 @@ class DatabaseManager {
}

func insertFrame(activeApplicationName: String?) -> Int64 {
// logger.debug("inserting frame: \(self.lastFrameId + 1) at offset: \(self.currentFrameOffset)")
let insert = frames.insert(chunkId <- currentChunkId, timestamp <- Date(), offsetIndex <- currentFrameOffset, self.activeApplicationName <- activeApplicationName)
let id = try! db.run(insert)
currentFrameOffset += 1
Expand All @@ -151,8 +157,8 @@ class DatabaseManager {
return (frame[offsetIndex], frame[filePath])
}

// let justFrameQuery = frames.filter(frames[id] === index).limit(1)
// try! db.run(justFrameQuery.delete())
// let justFrameQuery = frames.filter(frames[id] === index).limit(1)
// try! db.run(justFrameQuery.delete())
} catch {
return nil
}
Expand Down
49 changes: 49 additions & 0 deletions rem/ImageHelper.swift
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
//
// ImageHelper.swift
// rem
//
// Created by Jason McGhee on 12/31/23.
//

import Foundation
import os
import SwiftUI

class ImageHelper {
private static let logger = Logger(
subsystem: Bundle.main.bundleIdentifier!,
category: String(describing: ImageHelper.self)
)

// Useful for debugging...
static func pngData(from nsImage: NSImage) -> Data? {
guard let tiffRepresentation = nsImage.tiffRepresentation,
let bitmapImage = NSBitmapImageRep(data: tiffRepresentation) else {
logger.error("Failed to get TIFF representation of NSImage")
return nil
}

guard let pngData = bitmapImage.representation(using: .png, properties: [:]) else {
logger.error("Failed to convert NSImage to PNG")
return nil
}

return pngData
}

static func saveNSImage(image: NSImage, path: String) {
let pngData = pngData(from: image)
do {
if let savedir = RemFileManager.shared.getSaveDir() {
let outputPath = savedir.appendingPathComponent("\(path).png").path
let fileURL = URL(fileURLWithPath: outputPath)
try pngData?.write(to: fileURL)
logger.info("PNG file written successfully")
} else {
logger.error("Error writing PNG file")
}
} catch {
logger.error("Error writing PNG file: \(error)")
}
}
}
3 changes: 0 additions & 3 deletions rem/TextMerger.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ class TextMerger {
static let shared = TextMerger()

func mergeTexts(texts: [String]) -> String {
if texts.count == 1 {
return texts[0]
}
var mergedText: String = ""
var linesSeen = Set<String>()
for text in texts {
Expand Down
133 changes: 63 additions & 70 deletions rem/TimelineView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ struct TimelineView: View {
)
@ObservedObject var viewModel: TimelineViewModel
@State private var imageAnalysis: ImageAnalysis?
@State private var frame: NSRect
@State private var lastAnalyzedIndex: Int64 = -1 // To track the last analyzed index
@State var customHostingView: CustomHostingView?

private var ocrDebouncer = Debouncer(delay: 1.0)
Expand All @@ -29,21 +27,22 @@ struct TimelineView: View {
self.viewModel = viewModel
self.settingsManager = settingsManager
self.onClose = onClose
_frame = State(initialValue: NSScreen.main?.visibleFrame ?? NSRect.zero)
_customHostingView = State(initialValue: nil)
}

var body: some View {
ZStack {
let frame = NSScreen.main?.frame ?? NSRect.zero
let image = DatabaseManager.shared.getImage(index: viewModel.currentFrameIndex)
let nsImage = image.flatMap { NSImage(cgImage: $0, size: NSSize(width: $0.width, height: $0.height)) }

CustomHostingControllerRepresentable(
settingsManager: settingsManager,
onClose: onClose,
image: nsImage,
analysis: $imageAnalysis,
frame: frame
analysis: imageAnalysis,
frame: frame,
timelineOpen: viewModel.timelineOpen
)
.frame(width: frame.width, height: frame.height)
.ignoresSafeArea(.all)
Expand All @@ -54,6 +53,15 @@ struct TimelineView: View {
analyzeCurrentImage()
}

if image == nil {
VStack(alignment: .center) {
Text("Nothing to remember, or missing frame (if missing, sorry, still alpha!)")
.padding()
.background(RoundedRectangle(cornerRadius: 10)
.fill(Color.white.opacity(0.1)))
}
}

}
.ignoresSafeArea(.all)
}
Expand All @@ -80,39 +88,6 @@ struct TimelineView: View {
}
}
}


// Useful for debugging...
func pngData(from nsImage: NSImage) -> Data? {
guard let tiffRepresentation = nsImage.tiffRepresentation,
let bitmapImage = NSBitmapImageRep(data: tiffRepresentation) else {
logger.error("Failed to get TIFF representation of NSImage")
return nil
}

guard let pngData = bitmapImage.representation(using: .png, properties: [:]) else {
logger.error("Failed to convert NSImage to PNG")
return nil
}

return pngData
}

func saveNSImage(image: NSImage, path: String) {
let pngData = pngData(from: image)
do {
if let savedir = RemFileManager.shared.getSaveDir() {
let outputPath = savedir.appendingPathComponent("\(path).png").path
let fileURL = URL(fileURLWithPath: outputPath)
try pngData?.write(to: fileURL)
logger.info("PNG file written successfully")
} else {
logger.error("Error writing PNG file")
}
} catch {
logger.error("Error writing PNG file: \(error)")
}
}
}

class CustomHostingView: NSHostingView<AnyView> {
Expand Down Expand Up @@ -142,11 +117,11 @@ class CustomHostingView: NSHostingView<AnyView> {

private func configureImageView(with image: NSImage, in frame: NSRect) {
imageView.image = image

imageView.imageScaling = .scaleAxesIndependently

// Configuring frame to account for the offset and scaling
let adjustedFrame = CGRect(x: 0, y: -24, width: frame.width, height: frame.height + 24)
imageView.frame = adjustedFrame
imageView.frame = CGRect(x: 0, y: 0, width: frame.width, height: frame.height)
}

private func setupOverlayView() {
Expand All @@ -169,8 +144,9 @@ struct CustomHostingControllerRepresentable: NSViewControllerRepresentable {
var settingsManager: SettingsManager
var onClose: () -> Void
var image: NSImage?
@Binding var analysis: ImageAnalysis?
var analysis: ImageAnalysis?
var frame: NSRect
var timelineOpen: Bool

func makeNSViewController(context: Context) -> CustomHostingViewController {
let viewController = CustomHostingViewController()
Expand All @@ -181,20 +157,25 @@ struct CustomHostingControllerRepresentable: NSViewControllerRepresentable {
}

func updateNSViewController(_ nsViewController: CustomHostingViewController, context: Context) {
nsViewController.updateContent(image: image, frame: frame, analysis: analysis)
if timelineOpen {
nsViewController.updateContent(image: image, frame: frame, analysis: analysis)
}
nsViewController.onClose = onClose
nsViewController.settingsManager = settingsManager
}
}

class CustomHostingViewController: NSViewController {
var settingsManager: SettingsManager?
var onClose: (() -> Void)? // Closure to handle thumbnail click
var onClose: (() -> Void)?
var customHostingView: CustomHostingView?
var interceptingView: CustomInterceptingView?
var hadImage: Bool = false

override func viewWillAppear() {
view.enterFullScreenMode(NSScreen.main!)
DispatchQueue.main.async {
self.view.window?.makeKey()
}
}

override func loadView() {
Expand All @@ -211,44 +192,48 @@ class CustomHostingViewController: NSViewController {
interceptingView = _interceptingView
}

func updateImage(_ image: NSImage?, frame: NSRect) {
if let image = image {
// Image available: update or create CustomHostingView with the image
if customHostingView == nil {
customHostingView = CustomHostingView(image: image, frame: frame)
customHostingView?.frame = CGRect(origin: .zero, size: frame.size)
view.addSubview(customHostingView!)
} else {
customHostingView?.updateImage(image)
}
func updateImage(_ image: NSImage, frame: NSRect) {
// Image available: update or create CustomHostingView with the image
if customHostingView == nil {
customHostingView = CustomHostingView(image: image, frame: frame)
customHostingView?.frame = CGRect(origin: .zero, size: frame.size)
view.addSubview(customHostingView!)
} else {
// Image not available: Display VisualEffectView
displayVisualEffectView()
customHostingView?.updateImage(image)
}
customHostingView?.frame = CGRect(origin: .zero, size: frame.size)
}

func updateContent(image: NSImage?, frame: NSRect, analysis: ImageAnalysis?) {
if let image = image {
// Image is available
updateImage(image, frame: frame)
if let im = image {
if !view.isInFullScreenMode {
DispatchQueue.main.async {
self.view.enterFullScreenMode(NSScreen.main!)
}
}
updateImage(im, frame: frame)
updateAnalysis(analysis)
hadImage = true
} else {
// Image is not available, display VisualEffectView
if view.isInFullScreenMode {
DispatchQueue.main.async {
self.view.exitFullScreenMode()
}
}
displayVisualEffectView()
hadImage = false
}
}

private func displayVisualEffectView() {
// Ensure previous content is removed
view.subviews.forEach { $0.removeFromSuperview() }
interceptingView?.subviews.forEach { $0.removeFromSuperview() }

let visualEffectView = VisualEffectView(material: .hudWindow, blendingMode: .behindWindow)
.frame(maxWidth: .infinity, maxHeight: .infinity)
NSHostingController(rootView: visualEffectView)
.view
.frame = view.bounds
view.addSubview(NSHostingController(rootView: visualEffectView).view)
let visualEffectView = NSVisualEffectView()
visualEffectView.material = .hudWindow
visualEffectView.blendingMode = .behindWindow
visualEffectView.frame = interceptingView?.bounds ?? NSRect.zero

interceptingView?.addSubview(visualEffectView)
}

func updateAnalysis(_ analysis: ImageAnalysis?) {
Expand Down Expand Up @@ -300,6 +285,7 @@ class TimelineViewModel: ObservableObject {
private var speedFactor: Double = 0.05 // Adjust this factor based on UX requirements
@Published var currentFrameContinuous: Double = 0.0
@Published var currentFrameIndex: Int64 = 0
@Published var timelineOpen: Bool = false
private var indexUpdateThrottle = Throttler(delay: 0.05)

init() {
Expand All @@ -326,8 +312,11 @@ class TimelineViewModel: ObservableObject {
}

func setIndexToLatest() {
self.currentFrameContinuous = Double(DatabaseManager.shared.getMaxFrame())
self.currentFrameIndex = Int64(currentFrameContinuous)
let maxFrame = DatabaseManager.shared.getMaxFrame()
DispatchQueue.main.async {
self.currentFrameContinuous = Double(maxFrame)
self.currentFrameIndex = maxFrame
}
}

func updateIndexSafely() {
Expand All @@ -336,4 +325,8 @@ class TimelineViewModel: ObservableObject {
self.currentFrameIndex = rounded
}
}

func setIsOpen(isOpen: Bool) {
timelineOpen = isOpen
}
}
Loading

0 comments on commit 11b2288

Please sign in to comment.