more proactive vlm agent features
This commit is contained in:
227
FloatingHUDView.swift
Normal file
227
FloatingHUDView.swift
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
// FloatingHUDView.swift — Content for the always-on-top focus HUD panel
|
||||||
|
|
||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
struct FloatingHUDView: View {
|
||||||
|
@Environment(SessionManager.self) private var session
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
VStack(alignment: .leading, spacing: 0) {
|
||||||
|
header
|
||||||
|
Divider()
|
||||||
|
content
|
||||||
|
}
|
||||||
|
.background(.ultraThinMaterial)
|
||||||
|
.clipShape(.rect(cornerRadius: 12))
|
||||||
|
.shadow(color: .black.opacity(0.25), radius: 12, x: 0, y: 4)
|
||||||
|
.frame(width: 320)
|
||||||
|
.animation(.spring(duration: 0.3), value: session.proactiveCard?.id)
|
||||||
|
.animation(.spring(duration: 0.3), value: session.isExecuting)
|
||||||
|
.animation(.spring(duration: 0.3), value: session.executorOutput?.title)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Header
|
||||||
|
|
||||||
|
private var header: some View {
|
||||||
|
HStack(spacing: 8) {
|
||||||
|
Image(systemName: "eye.fill")
|
||||||
|
.foregroundStyle(.blue)
|
||||||
|
.font(.caption)
|
||||||
|
|
||||||
|
Text(session.activeTask?.title ?? "Focus Session")
|
||||||
|
.font(.caption.bold())
|
||||||
|
.lineLimit(1)
|
||||||
|
|
||||||
|
Spacer()
|
||||||
|
|
||||||
|
// Pulse dot — green when capturing, orange when executing
|
||||||
|
if session.isExecuting {
|
||||||
|
Image(systemName: "bolt.fill")
|
||||||
|
.font(.caption2)
|
||||||
|
.foregroundStyle(.orange)
|
||||||
|
} else {
|
||||||
|
Circle()
|
||||||
|
.fill(session.isCapturing ? Color.green : Color.secondary.opacity(0.4))
|
||||||
|
.frame(width: 7, height: 7)
|
||||||
|
.overlay(
|
||||||
|
Circle()
|
||||||
|
.fill(session.isCapturing ? Color.green.opacity(0.3) : .clear)
|
||||||
|
.frame(width: 14, height: 14)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.padding(.horizontal, 14)
|
||||||
|
.padding(.vertical, 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Content
|
||||||
|
|
||||||
|
@ViewBuilder
|
||||||
|
private var content: some View {
|
||||||
|
// Executor output sticky card (highest priority — persists until dismissed)
|
||||||
|
if let output = session.executorOutput {
|
||||||
|
ExecutorOutputCard(title: output.title, content: output.content) {
|
||||||
|
session.executorOutput = nil
|
||||||
|
}
|
||||||
|
.transition(.move(edge: .top).combined(with: .opacity))
|
||||||
|
}
|
||||||
|
// Executing spinner
|
||||||
|
else if session.isExecuting {
|
||||||
|
HStack(spacing: 10) {
|
||||||
|
ProgressView()
|
||||||
|
.controlSize(.small)
|
||||||
|
Text("Executing action…")
|
||||||
|
.font(.caption)
|
||||||
|
.foregroundStyle(.orange)
|
||||||
|
}
|
||||||
|
.padding(14)
|
||||||
|
.transition(.opacity)
|
||||||
|
}
|
||||||
|
// Proactive friction card
|
||||||
|
else if let card = session.proactiveCard {
|
||||||
|
HUDCardView(card: card)
|
||||||
|
.transition(.move(edge: .top).combined(with: .opacity))
|
||||||
|
}
|
||||||
|
// Latest VLM summary (idle state)
|
||||||
|
else {
|
||||||
|
Text(session.latestVlmSummary ?? "Monitoring your screen…")
|
||||||
|
.font(.caption)
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
.fixedSize(horizontal: false, vertical: true)
|
||||||
|
.padding(14)
|
||||||
|
.transition(.opacity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - HUD Card (friction + proposed actions)
|
||||||
|
|
||||||
|
private struct HUDCardView: View {
|
||||||
|
let card: ProactiveCard
|
||||||
|
@Environment(SessionManager.self) private var session
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
VStack(alignment: .leading, spacing: 10) {
|
||||||
|
// Title row
|
||||||
|
HStack(alignment: .top, spacing: 8) {
|
||||||
|
Image(systemName: card.icon)
|
||||||
|
.foregroundStyle(.purple)
|
||||||
|
|
||||||
|
VStack(alignment: .leading, spacing: 3) {
|
||||||
|
Text(card.title)
|
||||||
|
.font(.caption.bold())
|
||||||
|
.foregroundStyle(.purple)
|
||||||
|
Text(bodyText)
|
||||||
|
.font(.caption)
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
.fixedSize(horizontal: false, vertical: true)
|
||||||
|
.lineLimit(3)
|
||||||
|
}
|
||||||
|
|
||||||
|
Spacer()
|
||||||
|
|
||||||
|
Button { session.dismissProactiveCard() } label: {
|
||||||
|
Image(systemName: "xmark")
|
||||||
|
.font(.caption2.bold())
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
}
|
||||||
|
.buttonStyle(.plain)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Action buttons — only for VLM friction with proposed actions
|
||||||
|
if case .vlmFriction(_, _, let actions) = card.source, !actions.isEmpty {
|
||||||
|
VStack(alignment: .leading, spacing: 6) {
|
||||||
|
ForEach(Array(actions.prefix(2).enumerated()), id: \.offset) { index, action in
|
||||||
|
Button {
|
||||||
|
session.approveProactiveCard(actionIndex: index)
|
||||||
|
} label: {
|
||||||
|
VStack(alignment: .leading, spacing: 2) {
|
||||||
|
Text(action.label)
|
||||||
|
.font(.caption.bold())
|
||||||
|
.lineLimit(2)
|
||||||
|
.multilineTextAlignment(.leading)
|
||||||
|
if let details = action.details, !details.isEmpty {
|
||||||
|
Text(details)
|
||||||
|
.font(.caption2)
|
||||||
|
.foregroundStyle(.purple.opacity(0.7))
|
||||||
|
.lineLimit(2)
|
||||||
|
.multilineTextAlignment(.leading)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.frame(maxWidth: .infinity, alignment: .leading)
|
||||||
|
.padding(.horizontal, 10)
|
||||||
|
.padding(.vertical, 6)
|
||||||
|
.background(Color.purple.opacity(0.12))
|
||||||
|
.clipShape(.rect(cornerRadius: 8))
|
||||||
|
}
|
||||||
|
.buttonStyle(.plain)
|
||||||
|
.foregroundStyle(.purple)
|
||||||
|
}
|
||||||
|
|
||||||
|
Button("Not now — I'm good") { session.dismissProactiveCard() }
|
||||||
|
.font(.caption)
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
.buttonStyle(.plain)
|
||||||
|
.padding(.top, 2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.padding(14)
|
||||||
|
.background(Color.purple.opacity(0.07))
|
||||||
|
}
|
||||||
|
|
||||||
|
private var bodyText: String {
|
||||||
|
switch card.source {
|
||||||
|
case .vlmFriction(_, let description, _):
|
||||||
|
return description ?? "I noticed something that might be slowing you down."
|
||||||
|
case .appSwitchLoop(let apps, let count):
|
||||||
|
return "You've switched between \(apps.joined(separator: " ↔ ")) \(count)× — are you stuck?"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Executor Output Sticky Card
|
||||||
|
|
||||||
|
private struct ExecutorOutputCard: View {
|
||||||
|
let title: String
|
||||||
|
let content: String
|
||||||
|
let onDismiss: () -> Void
|
||||||
|
|
||||||
|
var body: some View {
|
||||||
|
VStack(alignment: .leading, spacing: 8) {
|
||||||
|
HStack(spacing: 6) {
|
||||||
|
Image(systemName: "checkmark.circle.fill")
|
||||||
|
.foregroundStyle(.green)
|
||||||
|
.font(.caption)
|
||||||
|
Text(title)
|
||||||
|
.font(.caption.bold())
|
||||||
|
.foregroundStyle(.green)
|
||||||
|
.lineLimit(1)
|
||||||
|
Spacer()
|
||||||
|
Button { onDismiss() } label: {
|
||||||
|
Image(systemName: "xmark")
|
||||||
|
.font(.caption2.bold())
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
}
|
||||||
|
.buttonStyle(.plain)
|
||||||
|
}
|
||||||
|
|
||||||
|
ScrollView {
|
||||||
|
Text(content)
|
||||||
|
.font(.caption)
|
||||||
|
.foregroundStyle(.primary)
|
||||||
|
.fixedSize(horizontal: false, vertical: true)
|
||||||
|
.frame(maxWidth: .infinity, alignment: .leading)
|
||||||
|
}
|
||||||
|
.frame(maxHeight: 120)
|
||||||
|
|
||||||
|
Button("Dismiss") { onDismiss() }
|
||||||
|
.font(.caption)
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
|
.buttonStyle(.plain)
|
||||||
|
.frame(maxWidth: .infinity, alignment: .trailing)
|
||||||
|
}
|
||||||
|
.padding(14)
|
||||||
|
.background(Color.green.opacity(0.07))
|
||||||
|
}
|
||||||
|
}
|
||||||
78
FloatingPanel.swift
Normal file
78
FloatingPanel.swift
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
// FloatingPanel.swift — Always-on-top NSPanel that hosts the floating HUD
|
||||||
|
|
||||||
|
import AppKit
|
||||||
|
import SwiftUI
|
||||||
|
|
||||||
|
// MARK: - Panel subclass
|
||||||
|
|
||||||
|
final class FloatingPanel: NSPanel {
|
||||||
|
init() {
|
||||||
|
super.init(
|
||||||
|
contentRect: NSRect(x: 0, y: 0, width: 320, height: 140),
|
||||||
|
styleMask: [.titled, .fullSizeContentView, .nonactivatingPanel, .utilityWindow],
|
||||||
|
backing: .buffered,
|
||||||
|
defer: false
|
||||||
|
)
|
||||||
|
// Always float above other windows, including full-screen apps
|
||||||
|
level = .floating
|
||||||
|
collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary]
|
||||||
|
|
||||||
|
// Drag anywhere on the panel body
|
||||||
|
isMovableByWindowBackground = true
|
||||||
|
isMovable = true
|
||||||
|
|
||||||
|
// Hide the standard title bar chrome
|
||||||
|
titleVisibility = .hidden
|
||||||
|
titlebarAppearsTransparent = true
|
||||||
|
|
||||||
|
// Transparent background so the SwiftUI material shows through
|
||||||
|
backgroundColor = .clear
|
||||||
|
isOpaque = false
|
||||||
|
|
||||||
|
// Don't activate the app when clicked (user keeps focus on their work)
|
||||||
|
becomesKeyOnlyIfNeeded = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Controller
|
||||||
|
|
||||||
|
@MainActor
|
||||||
|
final class FloatingPanelController {
|
||||||
|
static let shared = FloatingPanelController()
|
||||||
|
|
||||||
|
private var panel: FloatingPanel?
|
||||||
|
|
||||||
|
private init() {}
|
||||||
|
|
||||||
|
func show(session: SessionManager) {
|
||||||
|
if panel == nil {
|
||||||
|
let p = FloatingPanel()
|
||||||
|
let hud = FloatingHUDView()
|
||||||
|
.environment(session)
|
||||||
|
p.contentView = NSHostingView(rootView: hud)
|
||||||
|
|
||||||
|
// Position: top-right of the main screen, just below the menu bar
|
||||||
|
if let screen = NSScreen.main {
|
||||||
|
let margin: CGFloat = 16
|
||||||
|
let x = screen.visibleFrame.maxX - 320 - margin
|
||||||
|
let y = screen.visibleFrame.maxY - 160 - margin
|
||||||
|
p.setFrameOrigin(NSPoint(x: x, y: y))
|
||||||
|
} else {
|
||||||
|
p.center()
|
||||||
|
}
|
||||||
|
|
||||||
|
panel = p
|
||||||
|
}
|
||||||
|
panel?.orderFront(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hide() {
|
||||||
|
panel?.orderOut(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Call when the session fully ends to release the panel
|
||||||
|
func close() {
|
||||||
|
panel?.close()
|
||||||
|
panel = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,10 +10,14 @@
|
|||||||
FF935B1E2F78A83100ED3330 /* SpeakerKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1D2F78A83100ED3330 /* SpeakerKit */; };
|
FF935B1E2F78A83100ED3330 /* SpeakerKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1D2F78A83100ED3330 /* SpeakerKit */; };
|
||||||
FF935B202F78A83100ED3330 /* TTSKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1F2F78A83100ED3330 /* TTSKit */; };
|
FF935B202F78A83100ED3330 /* TTSKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B1F2F78A83100ED3330 /* TTSKit */; };
|
||||||
FF935B222F78A83100ED3330 /* WhisperKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B212F78A83100ED3330 /* WhisperKit */; };
|
FF935B222F78A83100ED3330 /* WhisperKit in Frameworks */ = {isa = PBXBuildFile; productRef = FF935B212F78A83100ED3330 /* WhisperKit */; };
|
||||||
|
FF935B242F78D0AA00ED3330 /* FloatingPanel.swift in Sources */ = {isa = PBXBuildFile; fileRef = FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */; };
|
||||||
|
FF935B262F78D0BF00ED3330 /* FloatingHUDView.swift in Sources */ = {isa = PBXBuildFile; fileRef = FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */; };
|
||||||
/* End PBXBuildFile section */
|
/* End PBXBuildFile section */
|
||||||
|
|
||||||
/* Begin PBXFileReference section */
|
/* Begin PBXFileReference section */
|
||||||
FF3296C22F785B3300C734EB /* LockInBro.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = LockInBro.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
FF3296C22F785B3300C734EB /* LockInBro.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = LockInBro.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||||
|
FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FloatingPanel.swift; sourceTree = "<group>"; };
|
||||||
|
FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FloatingHUDView.swift; sourceTree = "<group>"; };
|
||||||
/* End PBXFileReference section */
|
/* End PBXFileReference section */
|
||||||
|
|
||||||
/* Begin PBXFileSystemSynchronizedRootGroup section */
|
/* Begin PBXFileSystemSynchronizedRootGroup section */
|
||||||
@@ -43,6 +47,8 @@
|
|||||||
children = (
|
children = (
|
||||||
FF3296C42F785B3300C734EB /* LockInBro */,
|
FF3296C42F785B3300C734EB /* LockInBro */,
|
||||||
FF3296C32F785B3300C734EB /* Products */,
|
FF3296C32F785B3300C734EB /* Products */,
|
||||||
|
FF935B232F78D0AA00ED3330 /* FloatingPanel.swift */,
|
||||||
|
FF935B252F78D0BF00ED3330 /* FloatingHUDView.swift */,
|
||||||
);
|
);
|
||||||
sourceTree = "<group>";
|
sourceTree = "<group>";
|
||||||
};
|
};
|
||||||
@@ -134,6 +140,8 @@
|
|||||||
isa = PBXSourcesBuildPhase;
|
isa = PBXSourcesBuildPhase;
|
||||||
buildActionMask = 2147483647;
|
buildActionMask = 2147483647;
|
||||||
files = (
|
files = (
|
||||||
|
FF935B262F78D0BF00ED3330 /* FloatingHUDView.swift in Sources */,
|
||||||
|
FF935B242F78D0AA00ED3330 /* FloatingPanel.swift in Sources */,
|
||||||
);
|
);
|
||||||
runOnlyForDeploymentPostprocessing = 0;
|
runOnlyForDeploymentPostprocessing = 0;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -35,30 +35,40 @@ struct ContentView: View {
|
|||||||
private var mainContent: some View {
|
private var mainContent: some View {
|
||||||
NavigationSplitView {
|
NavigationSplitView {
|
||||||
// Sidebar
|
// Sidebar
|
||||||
List(AppTab.allCases, id: \.self, selection: $selectedTab) { tab in
|
VStack(spacing: 0) {
|
||||||
Label(tab.rawValue, systemImage: tab.systemImage)
|
List(AppTab.allCases, id: \.self, selection: $selectedTab) { tab in
|
||||||
.badge(tab == .focusSession && session.isSessionActive ? "●" : nil)
|
Label(tab.rawValue, systemImage: tab.systemImage)
|
||||||
}
|
.badge(tab == .focusSession && session.isSessionActive ? "●" : nil)
|
||||||
.navigationSplitViewColumnWidth(min: 160, ideal: 180)
|
}
|
||||||
.navigationTitle("LockInBro")
|
|
||||||
.toolbar {
|
Divider()
|
||||||
ToolbarItem {
|
|
||||||
|
HStack {
|
||||||
Button {
|
Button {
|
||||||
showingSettings = true
|
showingSettings = true
|
||||||
} label: {
|
} label: {
|
||||||
Image(systemName: "gear")
|
Label("Settings", systemImage: "gear")
|
||||||
|
.font(.subheadline)
|
||||||
}
|
}
|
||||||
.help("Settings")
|
.buttonStyle(.plain)
|
||||||
}
|
.foregroundStyle(.secondary)
|
||||||
ToolbarItem {
|
|
||||||
|
Spacer()
|
||||||
|
|
||||||
Button {
|
Button {
|
||||||
auth.logout()
|
auth.logout()
|
||||||
} label: {
|
} label: {
|
||||||
Image(systemName: "rectangle.portrait.and.arrow.right")
|
Image(systemName: "rectangle.portrait.and.arrow.right")
|
||||||
}
|
}
|
||||||
|
.buttonStyle(.plain)
|
||||||
|
.foregroundStyle(.secondary)
|
||||||
.help("Sign out")
|
.help("Sign out")
|
||||||
}
|
}
|
||||||
|
.padding(.horizontal, 12)
|
||||||
|
.padding(.vertical, 10)
|
||||||
}
|
}
|
||||||
|
.navigationSplitViewColumnWidth(min: 160, ideal: 180)
|
||||||
|
.navigationTitle("LockInBro")
|
||||||
.sheet(isPresented: $showingSettings) {
|
.sheet(isPresented: $showingSettings) {
|
||||||
SettingsSheet(geminiApiKey: $geminiApiKey)
|
SettingsSheet(geminiApiKey: $geminiApiKey)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,57 +63,8 @@ struct FocusSessionView: View {
|
|||||||
|
|
||||||
Divider()
|
Divider()
|
||||||
|
|
||||||
// Argus pipeline status bar (debug — always visible when session is active)
|
|
||||||
if !session.argusStatus.isEmpty {
|
|
||||||
HStack(spacing: 6) {
|
|
||||||
Text(session.argusStatus)
|
|
||||||
.font(.caption2)
|
|
||||||
.foregroundStyle(.secondary)
|
|
||||||
.lineLimit(1)
|
|
||||||
Spacer()
|
|
||||||
}
|
|
||||||
.padding(.horizontal)
|
|
||||||
.padding(.vertical, 4)
|
|
||||||
.background(Color.secondary.opacity(0.06))
|
|
||||||
Divider()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Proactive action card (VLM friction or app-switch loop)
|
|
||||||
if let card = session.proactiveCard {
|
|
||||||
ProactiveCardView(card: card, onDismiss: {
|
|
||||||
withAnimation { session.proactiveCard = nil }
|
|
||||||
}, onApprove: { label in
|
|
||||||
session.approvedActionLabel = label
|
|
||||||
withAnimation { session.proactiveCard = nil }
|
|
||||||
// Clear toast after 4 seconds
|
|
||||||
Task {
|
|
||||||
try? await Task.sleep(for: .seconds(4))
|
|
||||||
session.approvedActionLabel = nil
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.padding(.horizontal)
|
|
||||||
.padding(.top, 10)
|
|
||||||
.transition(.move(edge: .top).combined(with: .opacity))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Approved action confirmation toast
|
|
||||||
if let label = session.approvedActionLabel {
|
|
||||||
HStack(spacing: 8) {
|
|
||||||
Image(systemName: "checkmark.circle.fill")
|
|
||||||
.foregroundStyle(.green)
|
|
||||||
Text("Approved: \(label)")
|
|
||||||
.font(.caption.bold())
|
|
||||||
.foregroundStyle(.green)
|
|
||||||
Spacer()
|
|
||||||
}
|
|
||||||
.padding(.horizontal)
|
|
||||||
.padding(.vertical, 6)
|
|
||||||
.background(Color.green.opacity(0.08))
|
|
||||||
.transition(.move(edge: .top).combined(with: .opacity))
|
|
||||||
}
|
|
||||||
|
|
||||||
ScrollView {
|
ScrollView {
|
||||||
VStack(alignment: .leading, spacing: 20) {
|
VStack(alignment: .leading, spacing: 16) {
|
||||||
|
|
||||||
// Current step card
|
// Current step card
|
||||||
if let step = session.currentStep {
|
if let step = session.currentStep {
|
||||||
|
|||||||
@@ -13,6 +13,13 @@ struct LockInBroApp: App {
|
|||||||
ContentView()
|
ContentView()
|
||||||
.environment(auth)
|
.environment(auth)
|
||||||
.environment(session)
|
.environment(session)
|
||||||
|
.onChange(of: session.isSessionActive) { _, isActive in
|
||||||
|
if isActive {
|
||||||
|
FloatingPanelController.shared.show(session: session)
|
||||||
|
} else {
|
||||||
|
FloatingPanelController.shared.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
.defaultSize(width: 840, height: 580)
|
.defaultSize(width: 840, height: 580)
|
||||||
|
|
||||||
|
|||||||
@@ -29,11 +29,15 @@ final class SessionManager {
|
|||||||
var proactiveCard: ProactiveCard?
|
var proactiveCard: ProactiveCard?
|
||||||
/// Set when the user approves a proposed action — shown as a confirmation toast
|
/// Set when the user approves a proposed action — shown as a confirmation toast
|
||||||
var approvedActionLabel: String?
|
var approvedActionLabel: String?
|
||||||
|
/// Latest one-sentence summary from the VLM, shown in the floating HUD
|
||||||
|
var latestVlmSummary: String?
|
||||||
|
/// True while the argus executor is running an approved action
|
||||||
|
var isExecuting: Bool = false
|
||||||
|
/// Result produced by the executor's output() tool — shown as a sticky card in the HUD
|
||||||
|
var executorOutput: (title: String, content: String)?
|
||||||
|
|
||||||
// Screenshot engine
|
// Screenshot engine
|
||||||
var isCapturing: Bool = false
|
var isCapturing: Bool = false
|
||||||
/// Live pipeline status shown in FocusSessionView (updated at each stage)
|
|
||||||
var argusStatus: String = ""
|
|
||||||
|
|
||||||
private var captureTask: Task<Void, Never>?
|
private var captureTask: Task<Void, Never>?
|
||||||
private let captureInterval: TimeInterval = 5.0
|
private let captureInterval: TimeInterval = 5.0
|
||||||
@@ -55,6 +59,10 @@ final class SessionManager {
|
|||||||
// Argus subprocess (device-side VLM)
|
// Argus subprocess (device-side VLM)
|
||||||
@ObservationIgnored private var argusProcess: Process?
|
@ObservationIgnored private var argusProcess: Process?
|
||||||
@ObservationIgnored private var argusReadTask: Task<Void, Never>?
|
@ObservationIgnored private var argusReadTask: Task<Void, Never>?
|
||||||
|
@ObservationIgnored private var argusStdinPipe: Pipe?
|
||||||
|
/// Whether the current proactive card came from VLM (needs argus stdin response) vs local heuristic
|
||||||
|
@ObservationIgnored private var proactiveCardNeedsArgusResponse = false
|
||||||
|
@ObservationIgnored private var proactiveCardTimer: Task<Void, Never>?
|
||||||
private let argusPythonPath = "/Users/joyzhuo/miniconda3/envs/gmr/bin/python3"
|
private let argusPythonPath = "/Users/joyzhuo/miniconda3/envs/gmr/bin/python3"
|
||||||
private let argusRepoPath = "/Users/joyzhuo/yhack/lockinbro-argus"
|
private let argusRepoPath = "/Users/joyzhuo/yhack/lockinbro-argus"
|
||||||
|
|
||||||
@@ -148,6 +156,11 @@ final class SessionManager {
|
|||||||
showingResumeCard = false
|
showingResumeCard = false
|
||||||
proactiveCard = nil
|
proactiveCard = nil
|
||||||
approvedActionLabel = nil
|
approvedActionLabel = nil
|
||||||
|
latestVlmSummary = nil
|
||||||
|
isExecuting = false
|
||||||
|
executorOutput = nil
|
||||||
|
proactiveCardTimer?.cancel()
|
||||||
|
proactiveCardTimer = nil
|
||||||
screenshotHistory = []
|
screenshotHistory = []
|
||||||
persistedSessionId = nil
|
persistedSessionId = nil
|
||||||
}
|
}
|
||||||
@@ -179,6 +192,49 @@ final class SessionManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MARK: - Proactive Card Lifecycle
|
||||||
|
|
||||||
|
/// Show a proactive card and start the 15-second auto-dismiss timer.
|
||||||
|
/// - Parameter vlmCard: Pass true when the card came from VLM so argus gets a stdin response on dismiss.
|
||||||
|
private func showProactiveCard(_ card: ProactiveCard, vlmCard: Bool = false) {
|
||||||
|
proactiveCardNeedsArgusResponse = vlmCard
|
||||||
|
proactiveCardTimer?.cancel()
|
||||||
|
withAnimation { proactiveCard = card }
|
||||||
|
|
||||||
|
proactiveCardTimer = Task { [weak self] in
|
||||||
|
try? await Task.sleep(for: .seconds(15))
|
||||||
|
guard !Task.isCancelled, let self else { return }
|
||||||
|
await MainActor.run { self.dismissProactiveCard() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dismiss the current card (user tapped "Not now" or 15s elapsed).
|
||||||
|
func dismissProactiveCard() {
|
||||||
|
proactiveCardTimer?.cancel()
|
||||||
|
proactiveCardTimer = nil
|
||||||
|
withAnimation { proactiveCard = nil }
|
||||||
|
if proactiveCardNeedsArgusResponse { sendArgusResponse(0) }
|
||||||
|
proactiveCardNeedsArgusResponse = false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Approve action at the given index (0-based). Argus stdin uses 1-based (1 = action 0).
|
||||||
|
func approveProactiveCard(actionIndex: Int) {
|
||||||
|
proactiveCardTimer?.cancel()
|
||||||
|
proactiveCardTimer = nil
|
||||||
|
withAnimation { proactiveCard = nil }
|
||||||
|
if proactiveCardNeedsArgusResponse {
|
||||||
|
sendArgusResponse(actionIndex + 1)
|
||||||
|
isExecuting = true
|
||||||
|
}
|
||||||
|
proactiveCardNeedsArgusResponse = false
|
||||||
|
}
|
||||||
|
|
||||||
|
private func sendArgusResponse(_ choice: Int) {
|
||||||
|
guard let pipe = argusStdinPipe,
|
||||||
|
let data = "\(choice)\n".data(using: .utf8) else { return }
|
||||||
|
try? pipe.fileHandleForWriting.write(contentsOf: data)
|
||||||
|
}
|
||||||
|
|
||||||
// MARK: - App Switch Observer
|
// MARK: - App Switch Observer
|
||||||
|
|
||||||
private func startAppObserver() {
|
private func startAppObserver() {
|
||||||
@@ -236,7 +292,7 @@ final class SessionManager {
|
|||||||
// Only trigger card during active session and when none is already showing
|
// Only trigger card during active session and when none is already showing
|
||||||
guard isSessionActive, proactiveCard == nil else { return }
|
guard isSessionActive, proactiveCard == nil else { return }
|
||||||
if let loop = detectRepetitiveLoop() {
|
if let loop = detectRepetitiveLoop() {
|
||||||
proactiveCard = ProactiveCard(source: .appSwitchLoop(apps: loop.apps, switchCount: loop.count))
|
showProactiveCard(ProactiveCard(source: .appSwitchLoop(apps: loop.apps, switchCount: loop.count)), vlmCard: false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -266,7 +322,6 @@ final class SessionManager {
|
|||||||
private func startArgus(session: FocusSession, task: AppTask?) {
|
private func startArgus(session: FocusSession, task: AppTask?) {
|
||||||
guard FileManager.default.fileExists(atPath: argusPythonPath),
|
guard FileManager.default.fileExists(atPath: argusPythonPath),
|
||||||
FileManager.default.fileExists(atPath: argusRepoPath) else {
|
FileManager.default.fileExists(atPath: argusRepoPath) else {
|
||||||
argusStatus = "⚠️ Argus not found — using fallback capture"
|
|
||||||
startCapture()
|
startCapture()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -303,7 +358,8 @@ final class SessionManager {
|
|||||||
"--vlm", "gemini",
|
"--vlm", "gemini",
|
||||||
"--jwt", jwt,
|
"--jwt", jwt,
|
||||||
"--backend-url", "https://wahwa.com/api/v1",
|
"--backend-url", "https://wahwa.com/api/v1",
|
||||||
"--swift-ipc"
|
"--swift-ipc",
|
||||||
|
"--execute" // enables agentic executor; Swift sends 0/1/2 via stdin
|
||||||
]
|
]
|
||||||
if !geminiKey.isEmpty {
|
if !geminiKey.isEmpty {
|
||||||
arguments += ["--gemini-key", geminiKey]
|
arguments += ["--gemini-key", geminiKey]
|
||||||
@@ -314,49 +370,47 @@ final class SessionManager {
|
|||||||
process.currentDirectoryURL = URL(fileURLWithPath: argusRepoPath)
|
process.currentDirectoryURL = URL(fileURLWithPath: argusRepoPath)
|
||||||
process.arguments = arguments
|
process.arguments = arguments
|
||||||
|
|
||||||
// Pipe stdout for RESULT: lines; redirect stderr so argus logs don't clutter console
|
// Pipe stdout for RESULT:/STATUS:/EXEC_OUTPUT: lines
|
||||||
|
// stderr is NOT captured — leaving it unset lets argus log to the system console
|
||||||
|
// without risk of the pipe buffer filling and blocking the process.
|
||||||
let stdoutPipe = Pipe()
|
let stdoutPipe = Pipe()
|
||||||
let stderrPipe = Pipe()
|
let stdinPipe = Pipe()
|
||||||
process.standardOutput = stdoutPipe
|
process.standardOutput = stdoutPipe
|
||||||
process.standardError = stderrPipe
|
process.standardInput = stdinPipe
|
||||||
|
|
||||||
do {
|
do {
|
||||||
try process.launch()
|
try process.run()
|
||||||
} catch {
|
} catch {
|
||||||
argusStatus = "⚠️ Argus failed to launch — using fallback capture"
|
|
||||||
startCapture()
|
startCapture()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
argusProcess = process
|
argusProcess = process
|
||||||
|
argusStdinPipe = stdinPipe
|
||||||
isCapturing = true
|
isCapturing = true
|
||||||
|
|
||||||
let taskLabel = task?.title ?? "session"
|
// Read RESULT:/STATUS:/EXEC_OUTPUT: lines from argus stdout in a background task
|
||||||
argusStatus = "🚀 Argus started — waiting for first screenshot…"
|
|
||||||
sendDebugNotification(title: "🚀 Argus VLM Started", body: "Screen monitoring active for \(taskLabel)")
|
|
||||||
|
|
||||||
// Read RESULT: lines from argus stdout in a background task
|
|
||||||
let fileHandle = stdoutPipe.fileHandleForReading
|
let fileHandle = stdoutPipe.fileHandleForReading
|
||||||
sendDebugNotification(title: "🚀 Argus VLM Started", body: "Screen monitoring active for \(taskLabel)")
|
|
||||||
|
|
||||||
argusReadTask = Task { [weak self] in
|
argusReadTask = Task { [weak self] in
|
||||||
do {
|
do {
|
||||||
for try await line in fileHandle.bytes.lines {
|
for try await line in fileHandle.bytes.lines {
|
||||||
guard let self, !Task.isCancelled else { break }
|
guard let self, !Task.isCancelled else { break }
|
||||||
|
|
||||||
if line.hasPrefix("STATUS:") {
|
if line.hasPrefix("RESULT:") {
|
||||||
let event = String(line.dropFirst("STATUS:".count))
|
|
||||||
await MainActor.run { self.handleArgusStatus(event) }
|
|
||||||
} else if line.hasPrefix("RESULT:") {
|
|
||||||
let jsonStr = String(line.dropFirst("RESULT:".count))
|
let jsonStr = String(line.dropFirst("RESULT:".count))
|
||||||
guard let data = jsonStr.data(using: .utf8) else { continue }
|
if let data = jsonStr.data(using: .utf8),
|
||||||
if let result = try? JSONDecoder().decode(DistractionAnalysisResponse.self, from: data) {
|
let result = try? JSONDecoder().decode(DistractionAnalysisResponse.self, from: data) {
|
||||||
await MainActor.run {
|
await MainActor.run { self.applyDistractionResult(result) }
|
||||||
let summary = result.vlmSummary ?? "no summary"
|
}
|
||||||
self.argusStatus = "✅ \(summary)"
|
} else if line.hasPrefix("STATUS:exec_done:") {
|
||||||
self.sendDebugNotification(title: "✅ VLM Result", body: summary)
|
await MainActor.run { self.isExecuting = false }
|
||||||
self.applyDistractionResult(result)
|
} else if line.hasPrefix("EXEC_OUTPUT:") {
|
||||||
}
|
let jsonStr = String(line.dropFirst("EXEC_OUTPUT:".count))
|
||||||
|
if let data = jsonStr.data(using: .utf8),
|
||||||
|
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: String],
|
||||||
|
let title = obj["title"], let content = obj["content"] {
|
||||||
|
await MainActor.run { self.executorOutput = (title: title, content: content) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -369,6 +423,7 @@ final class SessionManager {
|
|||||||
private func stopArgus() {
|
private func stopArgus() {
|
||||||
argusReadTask?.cancel()
|
argusReadTask?.cancel()
|
||||||
argusReadTask = nil
|
argusReadTask = nil
|
||||||
|
argusStdinPipe = nil
|
||||||
if let proc = argusProcess {
|
if let proc = argusProcess {
|
||||||
proc.terminate()
|
proc.terminate()
|
||||||
argusProcess = nil
|
argusProcess = nil
|
||||||
@@ -482,6 +537,9 @@ final class SessionManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private func applyDistractionResult(_ result: DistractionAnalysisResponse) {
|
private func applyDistractionResult(_ result: DistractionAnalysisResponse) {
|
||||||
|
// 0. Store latest summary for the floating HUD
|
||||||
|
if let summary = result.vlmSummary { latestVlmSummary = summary }
|
||||||
|
|
||||||
// 1. Apply step side-effects (always)
|
// 1. Apply step side-effects (always)
|
||||||
for completedId in result.stepsCompleted {
|
for completedId in result.stepsCompleted {
|
||||||
if let idx = activeSteps.firstIndex(where: { $0.id == completedId }) {
|
if let idx = activeSteps.firstIndex(where: { $0.id == completedId }) {
|
||||||
@@ -506,11 +564,11 @@ final class SessionManager {
|
|||||||
// Task resumption detected — auto-surface resume card without button press
|
// Task resumption detected — auto-surface resume card without button press
|
||||||
Task { await fetchResumeCard() }
|
Task { await fetchResumeCard() }
|
||||||
} else if proactiveCard == nil {
|
} else if proactiveCard == nil {
|
||||||
proactiveCard = ProactiveCard(source: .vlmFriction(
|
showProactiveCard(ProactiveCard(source: .vlmFriction(
|
||||||
frictionType: friction.type,
|
frictionType: friction.type,
|
||||||
description: friction.description,
|
description: friction.description,
|
||||||
actions: friction.proposedActions
|
actions: friction.proposedActions
|
||||||
))
|
)), vlmCard: true)
|
||||||
}
|
}
|
||||||
} else if !result.onTask, result.confidence > 0.7, let nudge = result.gentleNudge {
|
} else if !result.onTask, result.confidence > 0.7, let nudge = result.gentleNudge {
|
||||||
// Only nudge if VLM found no actionable friction
|
// Only nudge if VLM found no actionable friction
|
||||||
@@ -522,34 +580,6 @@ final class SessionManager {
|
|||||||
|
|
||||||
// MARK: - Notifications
|
// MARK: - Notifications
|
||||||
|
|
||||||
private func handleArgusStatus(_ event: String) {
|
|
||||||
switch event {
|
|
||||||
case "screenshot_captured":
|
|
||||||
argusStatus = "📸 Screenshot captured — sending to VLM…"
|
|
||||||
sendDebugNotification(title: "📸 Screenshot Captured", body: "Sending to VLM for analysis…")
|
|
||||||
case "vlm_running":
|
|
||||||
argusStatus = "🤖 VLM analyzing screen…"
|
|
||||||
sendDebugNotification(title: "🤖 VLM Running", body: "Gemini is analyzing your screen…")
|
|
||||||
case "vlm_done":
|
|
||||||
argusStatus = "🧠 VLM done — applying result…"
|
|
||||||
sendDebugNotification(title: "🧠 VLM Done", body: "Analysis complete, processing result…")
|
|
||||||
default:
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private func sendDebugNotification(title: String, body: String) {
|
|
||||||
let content = UNMutableNotificationContent()
|
|
||||||
content.title = title
|
|
||||||
content.body = body
|
|
||||||
let req = UNNotificationRequest(
|
|
||||||
identifier: "debug-\(UUID().uuidString)",
|
|
||||||
content: content,
|
|
||||||
trigger: nil
|
|
||||||
)
|
|
||||||
UNUserNotificationCenter.current().add(req)
|
|
||||||
}
|
|
||||||
|
|
||||||
private func sendNudgeNotification(_ nudge: String) {
|
private func sendNudgeNotification(_ nudge: String) {
|
||||||
let content = UNMutableNotificationContent()
|
let content = UNMutableNotificationContent()
|
||||||
content.title = "Hey, quick check-in!"
|
content.title = "Hey, quick check-in!"
|
||||||
|
|||||||
Reference in New Issue
Block a user