more proactive vlm agent features

This commit is contained in:
joyzhuo
2026-03-29 00:58:22 -04:00
parent 15943b4759
commit 275a53ab40
7 changed files with 431 additions and 120 deletions

227
FloatingHUDView.swift Normal file
View File

@@ -0,0 +1,227 @@
// FloatingHUDView.swift Content for the always-on-top focus HUD panel
import SwiftUI
struct FloatingHUDView: View {
@Environment(SessionManager.self) private var session
var body: some View {
VStack(alignment: .leading, spacing: 0) {
header
Divider()
content
}
.background(.ultraThinMaterial)
.clipShape(.rect(cornerRadius: 12))
.shadow(color: .black.opacity(0.25), radius: 12, x: 0, y: 4)
.frame(width: 320)
.animation(.spring(duration: 0.3), value: session.proactiveCard?.id)
.animation(.spring(duration: 0.3), value: session.isExecuting)
.animation(.spring(duration: 0.3), value: session.executorOutput?.title)
}
// MARK: - Header
private var header: some View {
HStack(spacing: 8) {
Image(systemName: "eye.fill")
.foregroundStyle(.blue)
.font(.caption)
Text(session.activeTask?.title ?? "Focus Session")
.font(.caption.bold())
.lineLimit(1)
Spacer()
// Pulse dot green when capturing, orange when executing
if session.isExecuting {
Image(systemName: "bolt.fill")
.font(.caption2)
.foregroundStyle(.orange)
} else {
Circle()
.fill(session.isCapturing ? Color.green : Color.secondary.opacity(0.4))
.frame(width: 7, height: 7)
.overlay(
Circle()
.fill(session.isCapturing ? Color.green.opacity(0.3) : .clear)
.frame(width: 14, height: 14)
)
}
}
.padding(.horizontal, 14)
.padding(.vertical, 10)
}
// MARK: - Content
@ViewBuilder
private var content: some View {
// Executor output sticky card (highest priority persists until dismissed)
if let output = session.executorOutput {
ExecutorOutputCard(title: output.title, content: output.content) {
session.executorOutput = nil
}
.transition(.move(edge: .top).combined(with: .opacity))
}
// Executing spinner
else if session.isExecuting {
HStack(spacing: 10) {
ProgressView()
.controlSize(.small)
Text("Executing action…")
.font(.caption)
.foregroundStyle(.orange)
}
.padding(14)
.transition(.opacity)
}
// Proactive friction card
else if let card = session.proactiveCard {
HUDCardView(card: card)
.transition(.move(edge: .top).combined(with: .opacity))
}
// Latest VLM summary (idle state)
else {
Text(session.latestVlmSummary ?? "Monitoring your screen…")
.font(.caption)
.foregroundStyle(.secondary)
.fixedSize(horizontal: false, vertical: true)
.padding(14)
.transition(.opacity)
}
}
}
// MARK: - HUD Card (friction + proposed actions)
private struct HUDCardView: View {
let card: ProactiveCard
@Environment(SessionManager.self) private var session
var body: some View {
VStack(alignment: .leading, spacing: 10) {
// Title row
HStack(alignment: .top, spacing: 8) {
Image(systemName: card.icon)
.foregroundStyle(.purple)
VStack(alignment: .leading, spacing: 3) {
Text(card.title)
.font(.caption.bold())
.foregroundStyle(.purple)
Text(bodyText)
.font(.caption)
.foregroundStyle(.secondary)
.fixedSize(horizontal: false, vertical: true)
.lineLimit(3)
}
Spacer()
Button { session.dismissProactiveCard() } label: {
Image(systemName: "xmark")
.font(.caption2.bold())
.foregroundStyle(.secondary)
}
.buttonStyle(.plain)
}
// Action buttons only for VLM friction with proposed actions
if case .vlmFriction(_, _, let actions) = card.source, !actions.isEmpty {
VStack(alignment: .leading, spacing: 6) {
ForEach(Array(actions.prefix(2).enumerated()), id: \.offset) { index, action in
Button {
session.approveProactiveCard(actionIndex: index)
} label: {
VStack(alignment: .leading, spacing: 2) {
Text(action.label)
.font(.caption.bold())
.lineLimit(2)
.multilineTextAlignment(.leading)
if let details = action.details, !details.isEmpty {
Text(details)
.font(.caption2)
.foregroundStyle(.purple.opacity(0.7))
.lineLimit(2)
.multilineTextAlignment(.leading)
}
}
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.horizontal, 10)
.padding(.vertical, 6)
.background(Color.purple.opacity(0.12))
.clipShape(.rect(cornerRadius: 8))
}
.buttonStyle(.plain)
.foregroundStyle(.purple)
}
Button("Not now — I'm good") { session.dismissProactiveCard() }
.font(.caption)
.foregroundStyle(.secondary)
.buttonStyle(.plain)
.padding(.top, 2)
}
}
}
.padding(14)
.background(Color.purple.opacity(0.07))
}
private var bodyText: String {
switch card.source {
case .vlmFriction(_, let description, _):
return description ?? "I noticed something that might be slowing you down."
case .appSwitchLoop(let apps, let count):
return "You've switched between \(apps.joined(separator: "")) \(count)× — are you stuck?"
}
}
}
// MARK: - Executor Output Sticky Card
private struct ExecutorOutputCard: View {
let title: String
let content: String
let onDismiss: () -> Void
var body: some View {
VStack(alignment: .leading, spacing: 8) {
HStack(spacing: 6) {
Image(systemName: "checkmark.circle.fill")
.foregroundStyle(.green)
.font(.caption)
Text(title)
.font(.caption.bold())
.foregroundStyle(.green)
.lineLimit(1)
Spacer()
Button { onDismiss() } label: {
Image(systemName: "xmark")
.font(.caption2.bold())
.foregroundStyle(.secondary)
}
.buttonStyle(.plain)
}
ScrollView {
Text(content)
.font(.caption)
.foregroundStyle(.primary)
.fixedSize(horizontal: false, vertical: true)
.frame(maxWidth: .infinity, alignment: .leading)
}
.frame(maxHeight: 120)
Button("Dismiss") { onDismiss() }
.font(.caption)
.foregroundStyle(.secondary)
.buttonStyle(.plain)
.frame(maxWidth: .infinity, alignment: .trailing)
}
.padding(14)
.background(Color.green.opacity(0.07))
}
}

78
FloatingPanel.swift Normal file
View File

@@ -0,0 +1,78 @@
// FloatingPanel.swift Always-on-top NSPanel that hosts the floating HUD
import AppKit
import SwiftUI
// MARK: - Panel subclass
final class FloatingPanel: NSPanel {
init() {
super.init(
contentRect: NSRect(x: 0, y: 0, width: 320, height: 140),
styleMask: [.titled, .fullSizeContentView, .nonactivatingPanel, .utilityWindow],
backing: .buffered,
defer: false
)
// Always float above other windows, including full-screen apps
level = .floating
collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary]
// Drag anywhere on the panel body
isMovableByWindowBackground = true
isMovable = true
// Hide the standard title bar chrome
titleVisibility = .hidden
titlebarAppearsTransparent = true
// Transparent background so the SwiftUI material shows through
backgroundColor = .clear
isOpaque = false
// Don't activate the app when clicked (user keeps focus on their work)
becomesKeyOnlyIfNeeded = true
}
}
// MARK: - Controller
@MainActor
final class FloatingPanelController {
static let shared = FloatingPanelController()
private var panel: FloatingPanel?
private init() {}
func show(session: SessionManager) {
if panel == nil {
let p = FloatingPanel()
let hud = FloatingHUDView()
.environment(session)
p.contentView = NSHostingView(rootView: hud)
// Position: top-right of the main screen, just below the menu bar
if let screen = NSScreen.main {
let margin: CGFloat = 16
let x = screen.visibleFrame.maxX - 320 - margin
let y = screen.visibleFrame.maxY - 160 - margin
p.setFrameOrigin(NSPoint(x: x, y: y))
} else {
p.center()
}
panel = p
}
panel?.orderFront(nil)
}
func hide() {
panel?.orderOut(nil)
}
/// Call when the session fully ends to release the panel
func close() {
panel?.close()
panel = nil
}
}

View File

@@ -10,10 +10,14 @@
FF935B1E2F78A83100ED3330 /* SpeakerKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1D2F78A83100ED3330 /* SpeakerKit */; }; FF935B1E2F78A83100ED3330 /* SpeakerKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1D2F78A83100ED3330 /* SpeakerKit */; };
FF935B202F78A83100ED3330 /* TTSKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1F2F78A83100ED3330 /* TTSKit */; }; FF935B202F78A83100ED3330 /* TTSKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1F2F78A83100ED3330 /* TTSKit */; };
FF935B222F78A83100ED3330 /* WhisperKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B212F78A83100ED3330 /* WhisperKit */; }; FF935B222F78A83100ED3330 /* WhisperKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B212F78A83100ED3330 /* WhisperKit */; };
FF935B242F78D0AA00ED3330 /* FloatingPanel.swift in Sources */ = {isa = PBXBuildFile; fileRef = FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */; };
FF935B262F78D0BF00ED3330 /* FloatingHUDView.swift in Sources */ = {isa = PBXBuildFile; fileRef = FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */; };
/* End PBXBuildFile section */ /* End PBXBuildFile section */
/* Begin PBXFileReference section */ /* Begin PBXFileReference section */
FF3296C22F785B3300C734EB /* LockInBro.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = LockInBro.app; sourceTree = BUILT_PRODUCTS_DIR; }; FF3296C22F785B3300C734EB /* LockInBro.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = LockInBro.app; sourceTree = BUILT_PRODUCTS_DIR; };
FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FloatingPanel.swift; sourceTree = "<group>"; };
FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FloatingHUDView.swift; sourceTree = "<group>"; };
/* End PBXFileReference section */ /* End PBXFileReference section */
/* Begin PBXFileSystemSynchronizedRootGroup section */ /* Begin PBXFileSystemSynchronizedRootGroup section */
@@ -43,6 +47,8 @@
children = ( children = (
FF3296C42F785B3300C734EB /* LockInBro */, FF3296C42F785B3300C734EB /* LockInBro */,
FF3296C32F785B3300C734EB /* Products */, FF3296C32F785B3300C734EB /* Products */,
FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */,
FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */,
); );
sourceTree = "<group>"; sourceTree = "<group>";
}; };
@@ -134,6 +140,8 @@
isa = PBXSourcesBuildPhase; isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647; buildActionMask = 2147483647;
files = ( files = (
FF935B262F78D0BF00ED3330 /* FloatingHUDView.swift in Sources */,
FF935B242F78D0AA00ED3330 /* FloatingPanel.swift in Sources */,
); );
runOnlyForDeploymentPostprocessing = 0; runOnlyForDeploymentPostprocessing = 0;
}; };

View File

@@ -35,30 +35,40 @@ struct ContentView: View {
private var mainContent: some View { private var mainContent: some View {
NavigationSplitView { NavigationSplitView {
// Sidebar // Sidebar
VStack(spacing: 0) {
List(AppTab.allCases, id: \.self, selection: $selectedTab) { tab in List(AppTab.allCases, id: \.self, selection: $selectedTab) { tab in
Label(tab.rawValue, systemImage: tab.systemImage) Label(tab.rawValue, systemImage: tab.systemImage)
.badge(tab == .focusSession && session.isSessionActive ? "" : nil) .badge(tab == .focusSession && session.isSessionActive ? "" : nil)
} }
.navigationSplitViewColumnWidth(min: 160, ideal: 180)
.navigationTitle("LockInBro") Divider()
.toolbar {
ToolbarItem { HStack {
Button { Button {
showingSettings = true showingSettings = true
} label: { } label: {
Image(systemName: "gear") Label("Settings", systemImage: "gear")
.font(.subheadline)
} }
.help("Settings") .buttonStyle(.plain)
} .foregroundStyle(.secondary)
ToolbarItem {
Spacer()
Button { Button {
auth.logout() auth.logout()
} label: { } label: {
Image(systemName: "rectangle.portrait.and.arrow.right") Image(systemName: "rectangle.portrait.and.arrow.right")
} }
.buttonStyle(.plain)
.foregroundStyle(.secondary)
.help("Sign out") .help("Sign out")
} }
.padding(.horizontal, 12)
.padding(.vertical, 10)
} }
.navigationSplitViewColumnWidth(min: 160, ideal: 180)
.navigationTitle("LockInBro")
.sheet(isPresented: $showingSettings) { .sheet(isPresented: $showingSettings) {
SettingsSheet(geminiApiKey: $geminiApiKey) SettingsSheet(geminiApiKey: $geminiApiKey)
} }

View File

@@ -63,57 +63,8 @@ struct FocusSessionView: View {
Divider() Divider()
// Argus pipeline status bar (debug always visible when session is active)
if !session.argusStatus.isEmpty {
HStack(spacing: 6) {
Text(session.argusStatus)
.font(.caption2)
.foregroundStyle(.secondary)
.lineLimit(1)
Spacer()
}
.padding(.horizontal)
.padding(.vertical, 4)
.background(Color.secondary.opacity(0.06))
Divider()
}
// Proactive action card (VLM friction or app-switch loop)
if let card = session.proactiveCard {
ProactiveCardView(card: card, onDismiss: {
withAnimation { session.proactiveCard = nil }
}, onApprove: { label in
session.approvedActionLabel = label
withAnimation { session.proactiveCard = nil }
// Clear toast after 4 seconds
Task {
try? await Task.sleep(for: .seconds(4))
session.approvedActionLabel = nil
}
})
.padding(.horizontal)
.padding(.top, 10)
.transition(.move(edge: .top).combined(with: .opacity))
}
// Approved action confirmation toast
if let label = session.approvedActionLabel {
HStack(spacing: 8) {
Image(systemName: "checkmark.circle.fill")
.foregroundStyle(.green)
Text("Approved: \(label)")
.font(.caption.bold())
.foregroundStyle(.green)
Spacer()
}
.padding(.horizontal)
.padding(.vertical, 6)
.background(Color.green.opacity(0.08))
.transition(.move(edge: .top).combined(with: .opacity))
}
ScrollView { ScrollView {
VStack(alignment: .leading, spacing: 20) { VStack(alignment: .leading, spacing: 16) {
// Current step card // Current step card
if let step = session.currentStep { if let step = session.currentStep {

View File

@@ -13,6 +13,13 @@ struct LockInBroApp: App {
ContentView() ContentView()
.environment(auth) .environment(auth)
.environment(session) .environment(session)
.onChange(of: session.isSessionActive) { _, isActive in
if isActive {
FloatingPanelController.shared.show(session: session)
} else {
FloatingPanelController.shared.close()
}
}
} }
.defaultSize(width: 840, height: 580) .defaultSize(width: 840, height: 580)

View File

@@ -29,11 +29,15 @@ final class SessionManager {
var proactiveCard: ProactiveCard? var proactiveCard: ProactiveCard?
/// Set when the user approves a proposed action shown as a confirmation toast /// Set when the user approves a proposed action shown as a confirmation toast
var approvedActionLabel: String? var approvedActionLabel: String?
/// Latest one-sentence summary from the VLM, shown in the floating HUD
var latestVlmSummary: String?
/// True while the argus executor is running an approved action
var isExecuting: Bool = false
/// Result produced by the executor's output() tool shown as a sticky card in the HUD
var executorOutput: (title: String, content: String)?
// Screenshot engine // Screenshot engine
var isCapturing: Bool = false var isCapturing: Bool = false
/// Live pipeline status shown in FocusSessionView (updated at each stage)
var argusStatus: String = ""
private var captureTask: Task<Void, Never>? private var captureTask: Task<Void, Never>?
private let captureInterval: TimeInterval = 5.0 private let captureInterval: TimeInterval = 5.0
@@ -55,6 +59,10 @@ final class SessionManager {
// Argus subprocess (device-side VLM) // Argus subprocess (device-side VLM)
@ObservationIgnored private var argusProcess: Process? @ObservationIgnored private var argusProcess: Process?
@ObservationIgnored private var argusReadTask: Task<Void, Never>? @ObservationIgnored private var argusReadTask: Task<Void, Never>?
@ObservationIgnored private var argusStdinPipe: Pipe?
/// Whether the current proactive card came from VLM (needs argus stdin response) vs local heuristic
@ObservationIgnored private var proactiveCardNeedsArgusResponse = false
@ObservationIgnored private var proactiveCardTimer: Task<Void, Never>?
private let argusPythonPath = "/Users/joyzhuo/miniconda3/envs/gmr/bin/python3" private let argusPythonPath = "/Users/joyzhuo/miniconda3/envs/gmr/bin/python3"
private let argusRepoPath = "/Users/joyzhuo/yhack/lockinbro-argus" private let argusRepoPath = "/Users/joyzhuo/yhack/lockinbro-argus"
@@ -148,6 +156,11 @@ final class SessionManager {
showingResumeCard = false showingResumeCard = false
proactiveCard = nil proactiveCard = nil
approvedActionLabel = nil approvedActionLabel = nil
latestVlmSummary = nil
isExecuting = false
executorOutput = nil
proactiveCardTimer?.cancel()
proactiveCardTimer = nil
screenshotHistory = [] screenshotHistory = []
persistedSessionId = nil persistedSessionId = nil
} }
@@ -179,6 +192,49 @@ final class SessionManager {
} }
} }
// MARK: - Proactive Card Lifecycle
/// Show a proactive card and start the 15-second auto-dismiss timer.
/// - Parameter vlmCard: Pass true when the card came from VLM so argus gets a stdin response on dismiss.
private func showProactiveCard(_ card: ProactiveCard, vlmCard: Bool = false) {
proactiveCardNeedsArgusResponse = vlmCard
proactiveCardTimer?.cancel()
withAnimation { proactiveCard = card }
proactiveCardTimer = Task { [weak self] in
try? await Task.sleep(for: .seconds(15))
guard !Task.isCancelled, let self else { return }
await MainActor.run { self.dismissProactiveCard() }
}
}
/// Dismiss the current card (user tapped "Not now" or 15s elapsed).
func dismissProactiveCard() {
proactiveCardTimer?.cancel()
proactiveCardTimer = nil
withAnimation { proactiveCard = nil }
if proactiveCardNeedsArgusResponse { sendArgusResponse(0) }
proactiveCardNeedsArgusResponse = false
}
/// Approve action at the given index (0-based). Argus stdin uses 1-based (1 = action 0).
func approveProactiveCard(actionIndex: Int) {
proactiveCardTimer?.cancel()
proactiveCardTimer = nil
withAnimation { proactiveCard = nil }
if proactiveCardNeedsArgusResponse {
sendArgusResponse(actionIndex + 1)
isExecuting = true
}
proactiveCardNeedsArgusResponse = false
}
private func sendArgusResponse(_ choice: Int) {
guard let pipe = argusStdinPipe,
let data = "\(choice)\n".data(using: .utf8) else { return }
try? pipe.fileHandleForWriting.write(contentsOf: data)
}
// MARK: - App Switch Observer // MARK: - App Switch Observer
private func startAppObserver() { private func startAppObserver() {
@@ -236,7 +292,7 @@ final class SessionManager {
// Only trigger card during active session and when none is already showing // Only trigger card during active session and when none is already showing
guard isSessionActive, proactiveCard == nil else { return } guard isSessionActive, proactiveCard == nil else { return }
if let loop = detectRepetitiveLoop() { if let loop = detectRepetitiveLoop() {
proactiveCard = ProactiveCard(source: .appSwitchLoop(apps: loop.apps, switchCount: loop.count)) showProactiveCard(ProactiveCard(source: .appSwitchLoop(apps: loop.apps, switchCount: loop.count)), vlmCard: false)
} }
} }
@@ -266,7 +322,6 @@ final class SessionManager {
private func startArgus(session: FocusSession, task: AppTask?) { private func startArgus(session: FocusSession, task: AppTask?) {
guard FileManager.default.fileExists(atPath: argusPythonPath), guard FileManager.default.fileExists(atPath: argusPythonPath),
FileManager.default.fileExists(atPath: argusRepoPath) else { FileManager.default.fileExists(atPath: argusRepoPath) else {
argusStatus = "⚠️ Argus not found — using fallback capture"
startCapture() startCapture()
return return
} }
@@ -303,7 +358,8 @@ final class SessionManager {
"--vlm", "gemini", "--vlm", "gemini",
"--jwt", jwt, "--jwt", jwt,
"--backend-url", "https://wahwa.com/api/v1", "--backend-url", "https://wahwa.com/api/v1",
"--swift-ipc" "--swift-ipc",
"--execute" // enables agentic executor; Swift sends 0/1/2 via stdin
] ]
if !geminiKey.isEmpty { if !geminiKey.isEmpty {
arguments += ["--gemini-key", geminiKey] arguments += ["--gemini-key", geminiKey]
@@ -314,49 +370,47 @@ final class SessionManager {
process.currentDirectoryURL = URL(fileURLWithPath: argusRepoPath) process.currentDirectoryURL = URL(fileURLWithPath: argusRepoPath)
process.arguments = arguments process.arguments = arguments
// Pipe stdout for RESULT: lines; redirect stderr so argus logs don't clutter console // Pipe stdout for RESULT:/STATUS:/EXEC_OUTPUT: lines
// stderr is NOT captured leaving it unset lets argus log to the system console
// without risk of the pipe buffer filling and blocking the process.
let stdoutPipe = Pipe() let stdoutPipe = Pipe()
let stderrPipe = Pipe() let stdinPipe = Pipe()
process.standardOutput = stdoutPipe process.standardOutput = stdoutPipe
process.standardError = stderrPipe process.standardInput = stdinPipe
do { do {
try process.launch() try process.run()
} catch { } catch {
argusStatus = "⚠️ Argus failed to launch — using fallback capture"
startCapture() startCapture()
return return
} }
argusProcess = process argusProcess = process
argusStdinPipe = stdinPipe
isCapturing = true isCapturing = true
let taskLabel = task?.title ?? "session" // Read RESULT:/STATUS:/EXEC_OUTPUT: lines from argus stdout in a background task
argusStatus = "🚀 Argus started — waiting for first screenshot…"
sendDebugNotification(title: "🚀 Argus VLM Started", body: "Screen monitoring active for \(taskLabel)")
// Read RESULT: lines from argus stdout in a background task
let fileHandle = stdoutPipe.fileHandleForReading let fileHandle = stdoutPipe.fileHandleForReading
sendDebugNotification(title: "🚀 Argus VLM Started", body: "Screen monitoring active for \(taskLabel)")
argusReadTask = Task { [weak self] in argusReadTask = Task { [weak self] in
do { do {
for try await line in fileHandle.bytes.lines { for try await line in fileHandle.bytes.lines {
guard let self, !Task.isCancelled else { break } guard let self, !Task.isCancelled else { break }
if line.hasPrefix("STATUS:") { if line.hasPrefix("RESULT:") {
let event = String(line.dropFirst("STATUS:".count))
await MainActor.run { self.handleArgusStatus(event) }
} else if line.hasPrefix("RESULT:") {
let jsonStr = String(line.dropFirst("RESULT:".count)) let jsonStr = String(line.dropFirst("RESULT:".count))
guard let data = jsonStr.data(using: .utf8) else { continue } if let data = jsonStr.data(using: .utf8),
if let result = try? JSONDecoder().decode(DistractionAnalysisResponse.self, from: data) { let result = try? JSONDecoder().decode(DistractionAnalysisResponse.self, from: data) {
await MainActor.run { await MainActor.run { self.applyDistractionResult(result) }
let summary = result.vlmSummary ?? "no summary"
self.argusStatus = "\(summary)"
self.sendDebugNotification(title: "✅ VLM Result", body: summary)
self.applyDistractionResult(result)
} }
} else if line.hasPrefix("STATUS:exec_done:") {
await MainActor.run { self.isExecuting = false }
} else if line.hasPrefix("EXEC_OUTPUT:") {
let jsonStr = String(line.dropFirst("EXEC_OUTPUT:".count))
if let data = jsonStr.data(using: .utf8),
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: String],
let title = obj["title"], let content = obj["content"] {
await MainActor.run { self.executorOutput = (title: title, content: content) }
} }
} }
} }
@@ -369,6 +423,7 @@ final class SessionManager {
private func stopArgus() { private func stopArgus() {
argusReadTask?.cancel() argusReadTask?.cancel()
argusReadTask = nil argusReadTask = nil
argusStdinPipe = nil
if let proc = argusProcess { if let proc = argusProcess {
proc.terminate() proc.terminate()
argusProcess = nil argusProcess = nil
@@ -482,6 +537,9 @@ final class SessionManager {
} }
private func applyDistractionResult(_ result: DistractionAnalysisResponse) { private func applyDistractionResult(_ result: DistractionAnalysisResponse) {
// 0. Store latest summary for the floating HUD
if let summary = result.vlmSummary { latestVlmSummary = summary }
// 1. Apply step side-effects (always) // 1. Apply step side-effects (always)
for completedId in result.stepsCompleted { for completedId in result.stepsCompleted {
if let idx = activeSteps.firstIndex(where: { $0.id == completedId }) { if let idx = activeSteps.firstIndex(where: { $0.id == completedId }) {
@@ -506,11 +564,11 @@ final class SessionManager {
// Task resumption detected auto-surface resume card without button press // Task resumption detected auto-surface resume card without button press
Task { await fetchResumeCard() } Task { await fetchResumeCard() }
} else if proactiveCard == nil { } else if proactiveCard == nil {
proactiveCard = ProactiveCard(source: .vlmFriction( showProactiveCard(ProactiveCard(source: .vlmFriction(
frictionType: friction.type, frictionType: friction.type,
description: friction.description, description: friction.description,
actions: friction.proposedActions actions: friction.proposedActions
)) )), vlmCard: true)
} }
} else if !result.onTask, result.confidence > 0.7, let nudge = result.gentleNudge { } else if !result.onTask, result.confidence > 0.7, let nudge = result.gentleNudge {
// Only nudge if VLM found no actionable friction // Only nudge if VLM found no actionable friction
@@ -522,34 +580,6 @@ final class SessionManager {
// MARK: - Notifications // MARK: - Notifications
private func handleArgusStatus(_ event: String) {
switch event {
case "screenshot_captured":
argusStatus = "📸 Screenshot captured — sending to VLM…"
sendDebugNotification(title: "📸 Screenshot Captured", body: "Sending to VLM for analysis…")
case "vlm_running":
argusStatus = "🤖 VLM analyzing screen…"
sendDebugNotification(title: "🤖 VLM Running", body: "Gemini is analyzing your screen…")
case "vlm_done":
argusStatus = "🧠 VLM done — applying result…"
sendDebugNotification(title: "🧠 VLM Done", body: "Analysis complete, processing result…")
default:
break
}
}
private func sendDebugNotification(title: String, body: String) {
let content = UNMutableNotificationContent()
content.title = title
content.body = body
let req = UNNotificationRequest(
identifier: "debug-\(UUID().uuidString)",
content: content,
trigger: nil
)
UNUserNotificationCenter.current().add(req)
}
private func sendNudgeNotification(_ nudge: String) { private func sendNudgeNotification(_ nudge: String) {
let content = UNMutableNotificationContent() let content = UNMutableNotificationContent()
content.title = "Hey, quick check-in!" content.title = "Hey, quick check-in!"