more proactive vlm agent features
This commit is contained in:
227
FloatingHUDView.swift
Normal file
227
FloatingHUDView.swift
Normal file
@@ -0,0 +1,227 @@
|
||||
// FloatingHUDView.swift — Content for the always-on-top focus HUD panel
|
||||
|
||||
import SwiftUI
|
||||
|
||||
struct FloatingHUDView: View {
|
||||
@Environment(SessionManager.self) private var session
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
header
|
||||
Divider()
|
||||
content
|
||||
}
|
||||
.background(.ultraThinMaterial)
|
||||
.clipShape(.rect(cornerRadius: 12))
|
||||
.shadow(color: .black.opacity(0.25), radius: 12, x: 0, y: 4)
|
||||
.frame(width: 320)
|
||||
.animation(.spring(duration: 0.3), value: session.proactiveCard?.id)
|
||||
.animation(.spring(duration: 0.3), value: session.isExecuting)
|
||||
.animation(.spring(duration: 0.3), value: session.executorOutput?.title)
|
||||
}
|
||||
|
||||
// MARK: - Header
|
||||
|
||||
private var header: some View {
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "eye.fill")
|
||||
.foregroundStyle(.blue)
|
||||
.font(.caption)
|
||||
|
||||
Text(session.activeTask?.title ?? "Focus Session")
|
||||
.font(.caption.bold())
|
||||
.lineLimit(1)
|
||||
|
||||
Spacer()
|
||||
|
||||
// Pulse dot — green when capturing, orange when executing
|
||||
if session.isExecuting {
|
||||
Image(systemName: "bolt.fill")
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.orange)
|
||||
} else {
|
||||
Circle()
|
||||
.fill(session.isCapturing ? Color.green : Color.secondary.opacity(0.4))
|
||||
.frame(width: 7, height: 7)
|
||||
.overlay(
|
||||
Circle()
|
||||
.fill(session.isCapturing ? Color.green.opacity(0.3) : .clear)
|
||||
.frame(width: 14, height: 14)
|
||||
)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 10)
|
||||
}
|
||||
|
||||
// MARK: - Content
|
||||
|
||||
@ViewBuilder
|
||||
private var content: some View {
|
||||
// Executor output sticky card (highest priority — persists until dismissed)
|
||||
if let output = session.executorOutput {
|
||||
ExecutorOutputCard(title: output.title, content: output.content) {
|
||||
session.executorOutput = nil
|
||||
}
|
||||
.transition(.move(edge: .top).combined(with: .opacity))
|
||||
}
|
||||
// Executing spinner
|
||||
else if session.isExecuting {
|
||||
HStack(spacing: 10) {
|
||||
ProgressView()
|
||||
.controlSize(.small)
|
||||
Text("Executing action…")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.orange)
|
||||
}
|
||||
.padding(14)
|
||||
.transition(.opacity)
|
||||
}
|
||||
// Proactive friction card
|
||||
else if let card = session.proactiveCard {
|
||||
HUDCardView(card: card)
|
||||
.transition(.move(edge: .top).combined(with: .opacity))
|
||||
}
|
||||
// Latest VLM summary (idle state)
|
||||
else {
|
||||
Text(session.latestVlmSummary ?? "Monitoring your screen…")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
.padding(14)
|
||||
.transition(.opacity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - HUD Card (friction + proposed actions)
|
||||
|
||||
private struct HUDCardView: View {
|
||||
let card: ProactiveCard
|
||||
@Environment(SessionManager.self) private var session
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 10) {
|
||||
// Title row
|
||||
HStack(alignment: .top, spacing: 8) {
|
||||
Image(systemName: card.icon)
|
||||
.foregroundStyle(.purple)
|
||||
|
||||
VStack(alignment: .leading, spacing: 3) {
|
||||
Text(card.title)
|
||||
.font(.caption.bold())
|
||||
.foregroundStyle(.purple)
|
||||
Text(bodyText)
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
.lineLimit(3)
|
||||
}
|
||||
|
||||
Spacer()
|
||||
|
||||
Button { session.dismissProactiveCard() } label: {
|
||||
Image(systemName: "xmark")
|
||||
.font(.caption2.bold())
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
// Action buttons — only for VLM friction with proposed actions
|
||||
if case .vlmFriction(_, _, let actions) = card.source, !actions.isEmpty {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
ForEach(Array(actions.prefix(2).enumerated()), id: \.offset) { index, action in
|
||||
Button {
|
||||
session.approveProactiveCard(actionIndex: index)
|
||||
} label: {
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text(action.label)
|
||||
.font(.caption.bold())
|
||||
.lineLimit(2)
|
||||
.multilineTextAlignment(.leading)
|
||||
if let details = action.details, !details.isEmpty {
|
||||
Text(details)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.purple.opacity(0.7))
|
||||
.lineLimit(2)
|
||||
.multilineTextAlignment(.leading)
|
||||
}
|
||||
}
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.padding(.horizontal, 10)
|
||||
.padding(.vertical, 6)
|
||||
.background(Color.purple.opacity(0.12))
|
||||
.clipShape(.rect(cornerRadius: 8))
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.foregroundStyle(.purple)
|
||||
}
|
||||
|
||||
Button("Not now — I'm good") { session.dismissProactiveCard() }
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
.buttonStyle(.plain)
|
||||
.padding(.top, 2)
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(14)
|
||||
.background(Color.purple.opacity(0.07))
|
||||
}
|
||||
|
||||
private var bodyText: String {
|
||||
switch card.source {
|
||||
case .vlmFriction(_, let description, _):
|
||||
return description ?? "I noticed something that might be slowing you down."
|
||||
case .appSwitchLoop(let apps, let count):
|
||||
return "You've switched between \(apps.joined(separator: " ↔ ")) \(count)× — are you stuck?"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Executor Output Sticky Card
|
||||
|
||||
private struct ExecutorOutputCard: View {
|
||||
let title: String
|
||||
let content: String
|
||||
let onDismiss: () -> Void
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "checkmark.circle.fill")
|
||||
.foregroundStyle(.green)
|
||||
.font(.caption)
|
||||
Text(title)
|
||||
.font(.caption.bold())
|
||||
.foregroundStyle(.green)
|
||||
.lineLimit(1)
|
||||
Spacer()
|
||||
Button { onDismiss() } label: {
|
||||
Image(systemName: "xmark")
|
||||
.font(.caption2.bold())
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
ScrollView {
|
||||
Text(content)
|
||||
.font(.caption)
|
||||
.foregroundStyle(.primary)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
}
|
||||
.frame(maxHeight: 120)
|
||||
|
||||
Button("Dismiss") { onDismiss() }
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
.buttonStyle(.plain)
|
||||
.frame(maxWidth: .infinity, alignment: .trailing)
|
||||
}
|
||||
.padding(14)
|
||||
.background(Color.green.opacity(0.07))
|
||||
}
|
||||
}
|
||||
78
FloatingPanel.swift
Normal file
78
FloatingPanel.swift
Normal file
@@ -0,0 +1,78 @@
|
||||
// FloatingPanel.swift — Always-on-top NSPanel that hosts the floating HUD
|
||||
|
||||
import AppKit
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - Panel subclass
|
||||
|
||||
final class FloatingPanel: NSPanel {
|
||||
init() {
|
||||
super.init(
|
||||
contentRect: NSRect(x: 0, y: 0, width: 320, height: 140),
|
||||
styleMask: [.titled, .fullSizeContentView, .nonactivatingPanel, .utilityWindow],
|
||||
backing: .buffered,
|
||||
defer: false
|
||||
)
|
||||
// Always float above other windows, including full-screen apps
|
||||
level = .floating
|
||||
collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary]
|
||||
|
||||
// Drag anywhere on the panel body
|
||||
isMovableByWindowBackground = true
|
||||
isMovable = true
|
||||
|
||||
// Hide the standard title bar chrome
|
||||
titleVisibility = .hidden
|
||||
titlebarAppearsTransparent = true
|
||||
|
||||
// Transparent background so the SwiftUI material shows through
|
||||
backgroundColor = .clear
|
||||
isOpaque = false
|
||||
|
||||
// Don't activate the app when clicked (user keeps focus on their work)
|
||||
becomesKeyOnlyIfNeeded = true
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Controller
|
||||
|
||||
@MainActor
|
||||
final class FloatingPanelController {
|
||||
static let shared = FloatingPanelController()
|
||||
|
||||
private var panel: FloatingPanel?
|
||||
|
||||
private init() {}
|
||||
|
||||
func show(session: SessionManager) {
|
||||
if panel == nil {
|
||||
let p = FloatingPanel()
|
||||
let hud = FloatingHUDView()
|
||||
.environment(session)
|
||||
p.contentView = NSHostingView(rootView: hud)
|
||||
|
||||
// Position: top-right of the main screen, just below the menu bar
|
||||
if let screen = NSScreen.main {
|
||||
let margin: CGFloat = 16
|
||||
let x = screen.visibleFrame.maxX - 320 - margin
|
||||
let y = screen.visibleFrame.maxY - 160 - margin
|
||||
p.setFrameOrigin(NSPoint(x: x, y: y))
|
||||
} else {
|
||||
p.center()
|
||||
}
|
||||
|
||||
panel = p
|
||||
}
|
||||
panel?.orderFront(nil)
|
||||
}
|
||||
|
||||
func hide() {
|
||||
panel?.orderOut(nil)
|
||||
}
|
||||
|
||||
/// Call when the session fully ends to release the panel
|
||||
func close() {
|
||||
panel?.close()
|
||||
panel = nil
|
||||
}
|
||||
}
|
||||
@@ -10,10 +10,14 @@
|
||||
FF935B1E2F78A83100ED3330 /* SpeakerKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1D2F78A83100ED3330 /* SpeakerKit */; };
|
||||
FF935B202F78A83100ED3330 /* TTSKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1F2F78A83100ED3330 /* TTSKit */; };
|
||||
FF935B222F78A83100ED3330 /* WhisperKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B212F78A83100ED3330 /* WhisperKit */; };
|
||||
FF935B242F78D0AA00ED3330 /* FloatingPanel.swift in Sources */ = {isa = PBXBuildFile; fileRef = FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */; };
|
||||
FF935B262F78D0BF00ED3330 /* FloatingHUDView.swift in Sources */ = {isa = PBXBuildFile; fileRef = FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */; };
|
||||
/* End PBXBuildFile section */
|
||||
|
||||
/* Begin PBXFileReference section */
|
||||
FF3296C22F785B3300C734EB /* LockInBro.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = LockInBro.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FloatingPanel.swift; sourceTree = "<group>"; };
|
||||
FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FloatingHUDView.swift; sourceTree = "<group>"; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFileSystemSynchronizedRootGroup section */
|
||||
@@ -43,6 +47,8 @@
|
||||
children = (
|
||||
FF3296C42F785B3300C734EB /* LockInBro */,
|
||||
FF3296C32F785B3300C734EB /* Products */,
|
||||
FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */,
|
||||
FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */,
|
||||
);
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
@@ -134,6 +140,8 @@
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
FF935B262F78D0BF00ED3330 /* FloatingHUDView.swift in Sources */,
|
||||
FF935B242F78D0AA00ED3330 /* FloatingPanel.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
|
||||
@@ -35,30 +35,40 @@ struct ContentView: View {
|
||||
private var mainContent: some View {
|
||||
NavigationSplitView {
|
||||
// Sidebar
|
||||
VStack(spacing: 0) {
|
||||
List(AppTab.allCases, id: \.self, selection: $selectedTab) { tab in
|
||||
Label(tab.rawValue, systemImage: tab.systemImage)
|
||||
.badge(tab == .focusSession && session.isSessionActive ? "●" : nil)
|
||||
}
|
||||
.navigationSplitViewColumnWidth(min: 160, ideal: 180)
|
||||
.navigationTitle("LockInBro")
|
||||
.toolbar {
|
||||
ToolbarItem {
|
||||
|
||||
Divider()
|
||||
|
||||
HStack {
|
||||
Button {
|
||||
showingSettings = true
|
||||
} label: {
|
||||
Image(systemName: "gear")
|
||||
Label("Settings", systemImage: "gear")
|
||||
.font(.subheadline)
|
||||
}
|
||||
.help("Settings")
|
||||
}
|
||||
ToolbarItem {
|
||||
.buttonStyle(.plain)
|
||||
.foregroundStyle(.secondary)
|
||||
|
||||
Spacer()
|
||||
|
||||
Button {
|
||||
auth.logout()
|
||||
} label: {
|
||||
Image(systemName: "rectangle.portrait.and.arrow.right")
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.foregroundStyle(.secondary)
|
||||
.help("Sign out")
|
||||
}
|
||||
.padding(.horizontal, 12)
|
||||
.padding(.vertical, 10)
|
||||
}
|
||||
.navigationSplitViewColumnWidth(min: 160, ideal: 180)
|
||||
.navigationTitle("LockInBro")
|
||||
.sheet(isPresented: $showingSettings) {
|
||||
SettingsSheet(geminiApiKey: $geminiApiKey)
|
||||
}
|
||||
|
||||
@@ -63,57 +63,8 @@ struct FocusSessionView: View {
|
||||
|
||||
Divider()
|
||||
|
||||
// Argus pipeline status bar (debug — always visible when session is active)
|
||||
if !session.argusStatus.isEmpty {
|
||||
HStack(spacing: 6) {
|
||||
Text(session.argusStatus)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(.secondary)
|
||||
.lineLimit(1)
|
||||
Spacer()
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 4)
|
||||
.background(Color.secondary.opacity(0.06))
|
||||
Divider()
|
||||
}
|
||||
|
||||
// Proactive action card (VLM friction or app-switch loop)
|
||||
if let card = session.proactiveCard {
|
||||
ProactiveCardView(card: card, onDismiss: {
|
||||
withAnimation { session.proactiveCard = nil }
|
||||
}, onApprove: { label in
|
||||
session.approvedActionLabel = label
|
||||
withAnimation { session.proactiveCard = nil }
|
||||
// Clear toast after 4 seconds
|
||||
Task {
|
||||
try? await Task.sleep(for: .seconds(4))
|
||||
session.approvedActionLabel = nil
|
||||
}
|
||||
})
|
||||
.padding(.horizontal)
|
||||
.padding(.top, 10)
|
||||
.transition(.move(edge: .top).combined(with: .opacity))
|
||||
}
|
||||
|
||||
// Approved action confirmation toast
|
||||
if let label = session.approvedActionLabel {
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "checkmark.circle.fill")
|
||||
.foregroundStyle(.green)
|
||||
Text("Approved: \(label)")
|
||||
.font(.caption.bold())
|
||||
.foregroundStyle(.green)
|
||||
Spacer()
|
||||
}
|
||||
.padding(.horizontal)
|
||||
.padding(.vertical, 6)
|
||||
.background(Color.green.opacity(0.08))
|
||||
.transition(.move(edge: .top).combined(with: .opacity))
|
||||
}
|
||||
|
||||
ScrollView {
|
||||
VStack(alignment: .leading, spacing: 20) {
|
||||
VStack(alignment: .leading, spacing: 16) {
|
||||
|
||||
// Current step card
|
||||
if let step = session.currentStep {
|
||||
|
||||
@@ -13,6 +13,13 @@ struct LockInBroApp: App {
|
||||
ContentView()
|
||||
.environment(auth)
|
||||
.environment(session)
|
||||
.onChange(of: session.isSessionActive) { _, isActive in
|
||||
if isActive {
|
||||
FloatingPanelController.shared.show(session: session)
|
||||
} else {
|
||||
FloatingPanelController.shared.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
.defaultSize(width: 840, height: 580)
|
||||
|
||||
|
||||
@@ -29,11 +29,15 @@ final class SessionManager {
|
||||
var proactiveCard: ProactiveCard?
|
||||
/// Set when the user approves a proposed action — shown as a confirmation toast
|
||||
var approvedActionLabel: String?
|
||||
/// Latest one-sentence summary from the VLM, shown in the floating HUD
|
||||
var latestVlmSummary: String?
|
||||
/// True while the argus executor is running an approved action
|
||||
var isExecuting: Bool = false
|
||||
/// Result produced by the executor's output() tool — shown as a sticky card in the HUD
|
||||
var executorOutput: (title: String, content: String)?
|
||||
|
||||
// Screenshot engine
|
||||
var isCapturing: Bool = false
|
||||
/// Live pipeline status shown in FocusSessionView (updated at each stage)
|
||||
var argusStatus: String = ""
|
||||
|
||||
private var captureTask: Task<Void, Never>?
|
||||
private let captureInterval: TimeInterval = 5.0
|
||||
@@ -55,6 +59,10 @@ final class SessionManager {
|
||||
// Argus subprocess (device-side VLM)
|
||||
@ObservationIgnored private var argusProcess: Process?
|
||||
@ObservationIgnored private var argusReadTask: Task<Void, Never>?
|
||||
@ObservationIgnored private var argusStdinPipe: Pipe?
|
||||
/// Whether the current proactive card came from VLM (needs argus stdin response) vs local heuristic
|
||||
@ObservationIgnored private var proactiveCardNeedsArgusResponse = false
|
||||
@ObservationIgnored private var proactiveCardTimer: Task<Void, Never>?
|
||||
private let argusPythonPath = "/Users/joyzhuo/miniconda3/envs/gmr/bin/python3"
|
||||
private let argusRepoPath = "/Users/joyzhuo/yhack/lockinbro-argus"
|
||||
|
||||
@@ -148,6 +156,11 @@ final class SessionManager {
|
||||
showingResumeCard = false
|
||||
proactiveCard = nil
|
||||
approvedActionLabel = nil
|
||||
latestVlmSummary = nil
|
||||
isExecuting = false
|
||||
executorOutput = nil
|
||||
proactiveCardTimer?.cancel()
|
||||
proactiveCardTimer = nil
|
||||
screenshotHistory = []
|
||||
persistedSessionId = nil
|
||||
}
|
||||
@@ -179,6 +192,49 @@ final class SessionManager {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Proactive Card Lifecycle
|
||||
|
||||
/// Show a proactive card and start the 15-second auto-dismiss timer.
|
||||
/// - Parameter vlmCard: Pass true when the card came from VLM so argus gets a stdin response on dismiss.
|
||||
private func showProactiveCard(_ card: ProactiveCard, vlmCard: Bool = false) {
|
||||
proactiveCardNeedsArgusResponse = vlmCard
|
||||
proactiveCardTimer?.cancel()
|
||||
withAnimation { proactiveCard = card }
|
||||
|
||||
proactiveCardTimer = Task { [weak self] in
|
||||
try? await Task.sleep(for: .seconds(15))
|
||||
guard !Task.isCancelled, let self else { return }
|
||||
await MainActor.run { self.dismissProactiveCard() }
|
||||
}
|
||||
}
|
||||
|
||||
/// Dismiss the current card (user tapped "Not now" or 15s elapsed).
|
||||
func dismissProactiveCard() {
|
||||
proactiveCardTimer?.cancel()
|
||||
proactiveCardTimer = nil
|
||||
withAnimation { proactiveCard = nil }
|
||||
if proactiveCardNeedsArgusResponse { sendArgusResponse(0) }
|
||||
proactiveCardNeedsArgusResponse = false
|
||||
}
|
||||
|
||||
/// Approve action at the given index (0-based). Argus stdin uses 1-based (1 = action 0).
|
||||
func approveProactiveCard(actionIndex: Int) {
|
||||
proactiveCardTimer?.cancel()
|
||||
proactiveCardTimer = nil
|
||||
withAnimation { proactiveCard = nil }
|
||||
if proactiveCardNeedsArgusResponse {
|
||||
sendArgusResponse(actionIndex + 1)
|
||||
isExecuting = true
|
||||
}
|
||||
proactiveCardNeedsArgusResponse = false
|
||||
}
|
||||
|
||||
private func sendArgusResponse(_ choice: Int) {
|
||||
guard let pipe = argusStdinPipe,
|
||||
let data = "\(choice)\n".data(using: .utf8) else { return }
|
||||
try? pipe.fileHandleForWriting.write(contentsOf: data)
|
||||
}
|
||||
|
||||
// MARK: - App Switch Observer
|
||||
|
||||
private func startAppObserver() {
|
||||
@@ -236,7 +292,7 @@ final class SessionManager {
|
||||
// Only trigger card during active session and when none is already showing
|
||||
guard isSessionActive, proactiveCard == nil else { return }
|
||||
if let loop = detectRepetitiveLoop() {
|
||||
proactiveCard = ProactiveCard(source: .appSwitchLoop(apps: loop.apps, switchCount: loop.count))
|
||||
showProactiveCard(ProactiveCard(source: .appSwitchLoop(apps: loop.apps, switchCount: loop.count)), vlmCard: false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,7 +322,6 @@ final class SessionManager {
|
||||
private func startArgus(session: FocusSession, task: AppTask?) {
|
||||
guard FileManager.default.fileExists(atPath: argusPythonPath),
|
||||
FileManager.default.fileExists(atPath: argusRepoPath) else {
|
||||
argusStatus = "⚠️ Argus not found — using fallback capture"
|
||||
startCapture()
|
||||
return
|
||||
}
|
||||
@@ -303,7 +358,8 @@ final class SessionManager {
|
||||
"--vlm", "gemini",
|
||||
"--jwt", jwt,
|
||||
"--backend-url", "https://wahwa.com/api/v1",
|
||||
"--swift-ipc"
|
||||
"--swift-ipc",
|
||||
"--execute" // enables agentic executor; Swift sends 0/1/2 via stdin
|
||||
]
|
||||
if !geminiKey.isEmpty {
|
||||
arguments += ["--gemini-key", geminiKey]
|
||||
@@ -314,49 +370,47 @@ final class SessionManager {
|
||||
process.currentDirectoryURL = URL(fileURLWithPath: argusRepoPath)
|
||||
process.arguments = arguments
|
||||
|
||||
// Pipe stdout for RESULT: lines; redirect stderr so argus logs don't clutter console
|
||||
// Pipe stdout for RESULT:/STATUS:/EXEC_OUTPUT: lines
|
||||
// stderr is NOT captured — leaving it unset lets argus log to the system console
|
||||
// without risk of the pipe buffer filling and blocking the process.
|
||||
let stdoutPipe = Pipe()
|
||||
let stderrPipe = Pipe()
|
||||
let stdinPipe = Pipe()
|
||||
process.standardOutput = stdoutPipe
|
||||
process.standardError = stderrPipe
|
||||
process.standardInput = stdinPipe
|
||||
|
||||
do {
|
||||
try process.launch()
|
||||
try process.run()
|
||||
} catch {
|
||||
argusStatus = "⚠️ Argus failed to launch — using fallback capture"
|
||||
startCapture()
|
||||
return
|
||||
}
|
||||
|
||||
argusProcess = process
|
||||
argusStdinPipe = stdinPipe
|
||||
isCapturing = true
|
||||
|
||||
let taskLabel = task?.title ?? "session"
|
||||
argusStatus = "🚀 Argus started — waiting for first screenshot…"
|
||||
sendDebugNotification(title: "🚀 Argus VLM Started", body: "Screen monitoring active for \(taskLabel)")
|
||||
|
||||
// Read RESULT: lines from argus stdout in a background task
|
||||
// Read RESULT:/STATUS:/EXEC_OUTPUT: lines from argus stdout in a background task
|
||||
let fileHandle = stdoutPipe.fileHandleForReading
|
||||
sendDebugNotification(title: "🚀 Argus VLM Started", body: "Screen monitoring active for \(taskLabel)")
|
||||
|
||||
argusReadTask = Task { [weak self] in
|
||||
do {
|
||||
for try await line in fileHandle.bytes.lines {
|
||||
guard let self, !Task.isCancelled else { break }
|
||||
|
||||
if line.hasPrefix("STATUS:") {
|
||||
let event = String(line.dropFirst("STATUS:".count))
|
||||
await MainActor.run { self.handleArgusStatus(event) }
|
||||
} else if line.hasPrefix("RESULT:") {
|
||||
if line.hasPrefix("RESULT:") {
|
||||
let jsonStr = String(line.dropFirst("RESULT:".count))
|
||||
guard let data = jsonStr.data(using: .utf8) else { continue }
|
||||
if let result = try? JSONDecoder().decode(DistractionAnalysisResponse.self, from: data) {
|
||||
await MainActor.run {
|
||||
let summary = result.vlmSummary ?? "no summary"
|
||||
self.argusStatus = "✅ \(summary)"
|
||||
self.sendDebugNotification(title: "✅ VLM Result", body: summary)
|
||||
self.applyDistractionResult(result)
|
||||
if let data = jsonStr.data(using: .utf8),
|
||||
let result = try? JSONDecoder().decode(DistractionAnalysisResponse.self, from: data) {
|
||||
await MainActor.run { self.applyDistractionResult(result) }
|
||||
}
|
||||
} else if line.hasPrefix("STATUS:exec_done:") {
|
||||
await MainActor.run { self.isExecuting = false }
|
||||
} else if line.hasPrefix("EXEC_OUTPUT:") {
|
||||
let jsonStr = String(line.dropFirst("EXEC_OUTPUT:".count))
|
||||
if let data = jsonStr.data(using: .utf8),
|
||||
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: String],
|
||||
let title = obj["title"], let content = obj["content"] {
|
||||
await MainActor.run { self.executorOutput = (title: title, content: content) }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -369,6 +423,7 @@ final class SessionManager {
|
||||
private func stopArgus() {
|
||||
argusReadTask?.cancel()
|
||||
argusReadTask = nil
|
||||
argusStdinPipe = nil
|
||||
if let proc = argusProcess {
|
||||
proc.terminate()
|
||||
argusProcess = nil
|
||||
@@ -482,6 +537,9 @@ final class SessionManager {
|
||||
}
|
||||
|
||||
private func applyDistractionResult(_ result: DistractionAnalysisResponse) {
|
||||
// 0. Store latest summary for the floating HUD
|
||||
if let summary = result.vlmSummary { latestVlmSummary = summary }
|
||||
|
||||
// 1. Apply step side-effects (always)
|
||||
for completedId in result.stepsCompleted {
|
||||
if let idx = activeSteps.firstIndex(where: { $0.id == completedId }) {
|
||||
@@ -506,11 +564,11 @@ final class SessionManager {
|
||||
// Task resumption detected — auto-surface resume card without button press
|
||||
Task { await fetchResumeCard() }
|
||||
} else if proactiveCard == nil {
|
||||
proactiveCard = ProactiveCard(source: .vlmFriction(
|
||||
showProactiveCard(ProactiveCard(source: .vlmFriction(
|
||||
frictionType: friction.type,
|
||||
description: friction.description,
|
||||
actions: friction.proposedActions
|
||||
))
|
||||
)), vlmCard: true)
|
||||
}
|
||||
} else if !result.onTask, result.confidence > 0.7, let nudge = result.gentleNudge {
|
||||
// Only nudge if VLM found no actionable friction
|
||||
@@ -522,34 +580,6 @@ final class SessionManager {
|
||||
|
||||
// MARK: - Notifications
|
||||
|
||||
private func handleArgusStatus(_ event: String) {
|
||||
switch event {
|
||||
case "screenshot_captured":
|
||||
argusStatus = "📸 Screenshot captured — sending to VLM…"
|
||||
sendDebugNotification(title: "📸 Screenshot Captured", body: "Sending to VLM for analysis…")
|
||||
case "vlm_running":
|
||||
argusStatus = "🤖 VLM analyzing screen…"
|
||||
sendDebugNotification(title: "🤖 VLM Running", body: "Gemini is analyzing your screen…")
|
||||
case "vlm_done":
|
||||
argusStatus = "🧠 VLM done — applying result…"
|
||||
sendDebugNotification(title: "🧠 VLM Done", body: "Analysis complete, processing result…")
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
private func sendDebugNotification(title: String, body: String) {
|
||||
let content = UNMutableNotificationContent()
|
||||
content.title = title
|
||||
content.body = body
|
||||
let req = UNNotificationRequest(
|
||||
identifier: "debug-\(UUID().uuidString)",
|
||||
content: content,
|
||||
trigger: nil
|
||||
)
|
||||
UNUserNotificationCenter.current().add(req)
|
||||
}
|
||||
|
||||
private func sendNudgeNotification(_ nudge: String) {
|
||||
let content = UNMutableNotificationContent()
|
||||
content.title = "Hey, quick check-in!"
|
||||
|
||||
Reference in New Issue
Block a user