jcfs
6a46eb5a54
- Add task_agent_execution_logs table schema in SQLite with indexes - Create SQLiteMessageStore implementing MessageStore interface for persistent ACP message storage - Add task.logs WebSocket action for retrieving historical logs - Implement historical log replay on task.subscribe - Add HistoricalLogsProvider callback to WebSocket Hub Tests: - Add comprehensive unit tests for SQLiteMessageStore (29 tests) - Add integration tests for task.logs and historical replay (3 test suites) - Enhance e2e-test.sh with historical logs test scenarios (Steps 14-16) Files: - apps/backend/internal/orchestrator/acp/sqlite_store.go (new) - apps/backend/internal/orchestrator/acp/sqlite_store_test.go (new) - apps/backend/internal/task/repository/sqlite.go (schema) - apps/backend/internal/gateway/websocket/hub.go (provider) - apps/backend/internal/gateway/websocket/client.go (replay) - apps/backend/internal/orchestrator/wshandlers/handlers.go (task.logs) - apps/backend/cmd/kandev/main.go (wiring) - scripts/e2e-test.sh (test scenarios)
717 lines
26 KiB
Bash
Executable File
717 lines
26 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# End-to-End Test for Kandev
|
|
# This script tests the complete flow from task creation to agent execution
|
|
# using WebSocket-based communication with the backend.
|
|
#
|
|
# Prerequisites:
|
|
# - Docker running with kandev/augment-agent:latest image built
|
|
# - Augment session credentials in ~/.augment/session.json
|
|
# - Port 8080 available
|
|
# - websocat installed (cargo install websocat or brew install websocat)
|
|
#
|
|
# Usage: ./scripts/e2e-test.sh [--build]
|
|
# --build Build the Docker image before running tests
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
BACKEND_DIR="$PROJECT_ROOT/apps/backend"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Configuration
|
|
SERVER_PORT=8080
|
|
WS_URL="ws://localhost:${SERVER_PORT}/ws"
|
|
AGENTCTL_PORT=9999
|
|
WAIT_FOR_AGENT_SECONDS=120
|
|
SERVER_PID=""
|
|
BUILD_IMAGE=false
|
|
|
|
# Parse arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--build)
|
|
BUILD_IMAGE=true
|
|
shift
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Cleanup function
|
|
cleanup() {
|
|
echo -e "\n${YELLOW}Cleaning up...${NC}"
|
|
if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
|
|
kill "$SERVER_PID" 2>/dev/null || true
|
|
wait "$SERVER_PID" 2>/dev/null || true
|
|
fi
|
|
# Stop any running agent containers
|
|
docker ps --filter "name=kandev-agent" -q | xargs -r docker stop 2>/dev/null || true
|
|
docker ps --filter "name=kandev-agent" -aq | xargs -r docker rm 2>/dev/null || true
|
|
echo -e "${GREEN}Cleanup complete${NC}"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
# Helper functions
|
|
log_step() {
|
|
echo -e "\n${BLUE}=== $1 ===${NC}"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}✓ $1${NC}"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}✗ $1${NC}"
|
|
}
|
|
|
|
log_info() {
|
|
echo -e "${YELLOW}→ $1${NC}"
|
|
}
|
|
|
|
# WebSocket helper function
|
|
# Sends a request via WebSocket and returns the response
|
|
ws_request() {
|
|
local action=$1
|
|
local payload=$2
|
|
local id=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || echo "req-$$-$RANDOM")
|
|
|
|
# Compact the payload to single line (remove newlines and extra spaces)
|
|
local compact_payload=$(echo "$payload" | jq -c '.')
|
|
local message="{\"id\":\"${id}\",\"type\":\"request\",\"action\":\"${action}\",\"payload\":${compact_payload}}"
|
|
|
|
# Send request and get response (with timeout)
|
|
echo "$message" | timeout 10 websocat -n1 "${WS_URL}" 2>/dev/null
|
|
}
|
|
|
|
wait_for_server() {
|
|
log_info "Waiting for server to be ready..."
|
|
for i in {1..30}; do
|
|
HEALTH=$(ws_request "health.check" '{}' 2>/dev/null)
|
|
if echo "$HEALTH" | jq -e '.payload.status == "ok"' > /dev/null 2>&1; then
|
|
log_success "Server is ready"
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
log_error "Server failed to start"
|
|
exit 1
|
|
}
|
|
|
|
wait_for_agentctl() {
|
|
local container_ip=$1
|
|
log_info "Waiting for agentctl to be ready at ${container_ip}:${AGENTCTL_PORT}..."
|
|
for i in {1..30}; do
|
|
if curl -s "http://${container_ip}:${AGENTCTL_PORT}/health" > /dev/null 2>&1; then
|
|
log_success "agentctl is ready"
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
log_error "agentctl failed to start"
|
|
return 1
|
|
}
|
|
|
|
# Check prerequisites
|
|
log_step "Checking Prerequisites"
|
|
|
|
if ! command -v websocat &> /dev/null; then
|
|
log_error "websocat is not installed"
|
|
log_info "Install with: cargo install websocat"
|
|
log_info "Or on macOS: brew install websocat"
|
|
exit 1
|
|
fi
|
|
log_success "websocat is available"
|
|
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "Docker is not installed"
|
|
exit 1
|
|
fi
|
|
log_success "Docker is available"
|
|
|
|
if ! docker info &> /dev/null; then
|
|
log_error "Docker daemon is not running"
|
|
exit 1
|
|
fi
|
|
log_success "Docker daemon is running"
|
|
|
|
if [ ! -f "$HOME/.augment/session.json" ]; then
|
|
log_error "Augment session not found at ~/.augment/session.json"
|
|
log_info "Please log in with: auggie login"
|
|
exit 1
|
|
fi
|
|
log_success "Augment session credentials found"
|
|
|
|
# Build Docker image if requested or if it doesn't exist
|
|
if [ "$BUILD_IMAGE" = true ]; then
|
|
log_step "Building Docker Image"
|
|
log_info "Building kandev/augment-agent:latest with agentctl..."
|
|
cd "$BACKEND_DIR"
|
|
docker build -t kandev/augment-agent:latest -f dockerfiles/augment-agent/Dockerfile .
|
|
log_success "Docker image built"
|
|
elif ! docker image inspect kandev/augment-agent:latest &> /dev/null; then
|
|
log_error "Docker image kandev/augment-agent:latest not found"
|
|
log_info "Build it with: ./scripts/e2e-test.sh --build"
|
|
log_info "Or manually: cd apps/backend && docker build -t kandev/augment-agent:latest -f dockerfiles/augment-agent/Dockerfile ."
|
|
exit 1
|
|
else
|
|
log_success "Agent Docker image exists"
|
|
fi
|
|
|
|
# Kill any existing server on port 8080
|
|
if lsof -i :${SERVER_PORT} &> /dev/null; then
|
|
log_info "Killing existing process on port ${SERVER_PORT}"
|
|
pkill -9 -f "kandev" 2>/dev/null || true
|
|
sleep 2
|
|
fi
|
|
|
|
# Build and start the server
|
|
log_step "Building and Starting Server"
|
|
|
|
cd "$BACKEND_DIR"
|
|
#rm -f kandev.db kandev.db-shm kandev.db-wal
|
|
|
|
log_info "Building kandev..."
|
|
go build -o kandev ./cmd/kandev
|
|
|
|
log_info "Starting server..."
|
|
./kandev > /tmp/kandev-e2e.log 2>&1 &
|
|
SERVER_PID=$!
|
|
log_info "Server PID: $SERVER_PID"
|
|
|
|
wait_for_server
|
|
|
|
# Run the E2E test
|
|
log_step "Step 1: Create Board"
|
|
BOARD_RESPONSE=$(ws_request "board.create" '{"name": "E2E Test Board", "description": "Automated end-to-end test"}')
|
|
BOARD_ID=$(echo "$BOARD_RESPONSE" | jq -r '.payload.id')
|
|
if [ "$BOARD_ID" == "null" ] || [ -z "$BOARD_ID" ]; then
|
|
log_error "Failed to create board"
|
|
echo "$BOARD_RESPONSE"
|
|
exit 1
|
|
fi
|
|
log_success "Created board: $BOARD_ID"
|
|
|
|
log_step "Step 2: Create Column"
|
|
COLUMN_RESPONSE=$(ws_request "column.create" "{\"board_id\": \"${BOARD_ID}\", \"name\": \"To Do\", \"position\": 0}")
|
|
COLUMN_ID=$(echo "$COLUMN_RESPONSE" | jq -r '.payload.id')
|
|
if [ "$COLUMN_ID" == "null" ] || [ -z "$COLUMN_ID" ]; then
|
|
log_error "Failed to create column"
|
|
echo "$COLUMN_RESPONSE"
|
|
exit 1
|
|
fi
|
|
log_success "Created column: $COLUMN_ID"
|
|
|
|
log_step "Step 3: Create Task"
|
|
# Create a temp directory for the test workspace
|
|
TEST_WORKSPACE=$(mktemp -d)
|
|
log_info "Test workspace: $TEST_WORKSPACE"
|
|
|
|
# Create a simple task that creates a file
|
|
TASK_PAYLOAD=$(cat <<EOF
|
|
{
|
|
"title": "E2E Test Task",
|
|
"description": "Create a file named 'agent-result.txt' in the workspace root with the content 'Hello from Agent'. Do nothing else.",
|
|
"board_id": "${BOARD_ID}",
|
|
"column_id": "${COLUMN_ID}",
|
|
"repository_url": "${TEST_WORKSPACE}",
|
|
"agent_type": "augment-agent"
|
|
}
|
|
EOF
|
|
)
|
|
TASK_RESPONSE=$(ws_request "task.create" "$TASK_PAYLOAD")
|
|
TASK_ID=$(echo "$TASK_RESPONSE" | jq -r '.payload.id')
|
|
if [ "$TASK_ID" == "null" ] || [ -z "$TASK_ID" ]; then
|
|
log_error "Failed to create task"
|
|
echo "$TASK_RESPONSE"
|
|
exit 1
|
|
fi
|
|
log_success "Created task: $TASK_ID"
|
|
|
|
log_step "Step 4: Start Task via Orchestrator"
|
|
START_RESPONSE=$(ws_request "orchestrator.start" "{\"task_id\": \"${TASK_ID}\"}")
|
|
AGENT_ID=$(echo "$START_RESPONSE" | jq -r '.payload.agent_instance_id')
|
|
if [ "$AGENT_ID" == "null" ] || [ -z "$AGENT_ID" ]; then
|
|
log_error "Failed to start task"
|
|
echo "$START_RESPONSE"
|
|
exit 1
|
|
fi
|
|
log_success "Task started, agent ID: $AGENT_ID"
|
|
|
|
log_step "Step 5: Verify Agent Container Running"
|
|
sleep 3
|
|
CONTAINER_ID=$(docker ps --filter "name=kandev-agent" --format "{{.ID}}" | head -1)
|
|
if [ -z "$CONTAINER_ID" ]; then
|
|
log_error "No agent container running"
|
|
exit 1
|
|
fi
|
|
log_success "Agent container is running: $CONTAINER_ID"
|
|
|
|
# Get container IP for agentctl health check
|
|
CONTAINER_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_ID" 2>/dev/null)
|
|
if [ -n "$CONTAINER_IP" ]; then
|
|
log_info "Container IP: $CONTAINER_IP"
|
|
fi
|
|
|
|
log_step "Step 6: Verify agentctl is Running"
|
|
sleep 2
|
|
if [ -n "$CONTAINER_IP" ]; then
|
|
if wait_for_agentctl "$CONTAINER_IP"; then
|
|
# Check agentctl status
|
|
AGENTCTL_STATUS=$(curl -s "http://${CONTAINER_IP}:${AGENTCTL_PORT}/api/v1/status" 2>/dev/null || echo '{}')
|
|
AGENT_PROCESS_STATE=$(echo "$AGENTCTL_STATUS" | jq -r '.agent_status // "unknown"')
|
|
log_info "Agent process state: $AGENT_PROCESS_STATE"
|
|
else
|
|
log_info "Could not reach agentctl directly (may be in bridge network)"
|
|
fi
|
|
else
|
|
log_info "Container IP not available (using host network)"
|
|
fi
|
|
|
|
# Check server logs for agentctl communication
|
|
if grep -q "agentctl" /tmp/kandev-e2e.log; then
|
|
log_success "Backend is communicating with agentctl"
|
|
else
|
|
log_info "Checking backend logs..."
|
|
fi
|
|
|
|
if grep -q "agent started" /tmp/kandev-e2e.log || grep -q "initializing agent" /tmp/kandev-e2e.log; then
|
|
log_success "Agent initialization started"
|
|
else
|
|
log_info "Waiting for agent initialization..."
|
|
fi
|
|
|
|
log_step "Step 7: Wait for Agent to Complete First Prompt"
|
|
log_info "Waiting up to ${WAIT_FOR_AGENT_SECONDS} seconds for agent to become READY..."
|
|
AGENT_OUTPUT_FILE="${TEST_WORKSPACE}/agent-result.txt"
|
|
AGENT_READY=false
|
|
for i in $(seq 1 $WAIT_FOR_AGENT_SECONDS); do
|
|
# Check agent status - READY means prompt completed
|
|
AGENT_STATUS_RESPONSE=$(ws_request "agent.status" "{\"agent_id\": \"${AGENT_ID}\"}" 2>/dev/null || echo '{}')
|
|
AGENT_STATUS=$(echo "$AGENT_STATUS_RESPONSE" | jq -r '.payload.status // "UNKNOWN"')
|
|
|
|
if [ "$AGENT_STATUS" == "READY" ]; then
|
|
log_success "Agent is READY after $i seconds (prompt completed)"
|
|
AGENT_READY=true
|
|
break
|
|
elif [ "$AGENT_STATUS" == "FAILED" ]; then
|
|
log_error "Agent failed"
|
|
echo "$AGENT_STATUS_RESPONSE" | jq .
|
|
if [ -n "$CONTAINER_ID" ]; then
|
|
log_info "Container logs:"
|
|
docker logs "$CONTAINER_ID" 2>&1 | tail -30
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
# Check if container is still running
|
|
CONTAINER_RUNNING=$(docker ps --filter "id=${CONTAINER_ID}" -q | wc -l)
|
|
if [ "$CONTAINER_RUNNING" -eq 0 ]; then
|
|
log_info "Container stopped unexpectedly"
|
|
break
|
|
fi
|
|
|
|
if [ $((i % 10)) -eq 0 ]; then
|
|
log_info "Still waiting... ($i seconds, status: $AGENT_STATUS)"
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if [ "$AGENT_READY" != "true" ]; then
|
|
log_error "Agent did not become READY within timeout (last status: $AGENT_STATUS)"
|
|
exit 1
|
|
fi
|
|
|
|
log_step "Step 8: Verify Agent Response"
|
|
# Get container logs to verify agent processed the request
|
|
FINAL_CONTAINER_ID=$(docker ps -a --filter "name=kandev-agent" --format "{{.ID}}" | head -1)
|
|
if [ -n "$FINAL_CONTAINER_ID" ]; then
|
|
log_info "Checking container logs for $FINAL_CONTAINER_ID..."
|
|
CONTAINER_LOGS=$(docker logs "$FINAL_CONTAINER_ID" 2>&1 | tail -100)
|
|
|
|
# Check for agentctl activity
|
|
if echo "$CONTAINER_LOGS" | grep -q "agentctl"; then
|
|
log_success "agentctl was active in container"
|
|
fi
|
|
|
|
# Check for ACP messages
|
|
if echo "$CONTAINER_LOGS" | grep -q '"stopReason"'; then
|
|
log_success "Agent completed with stop reason"
|
|
elif echo "$CONTAINER_LOGS" | grep -q 'session/update'; then
|
|
log_success "Agent sent session updates"
|
|
elif echo "$CONTAINER_LOGS" | grep -q 'initialize'; then
|
|
log_success "Agent received initialize request"
|
|
else
|
|
log_info "Container logs (last 20 lines):"
|
|
echo "$CONTAINER_LOGS" | tail -20
|
|
fi
|
|
else
|
|
log_info "Container already removed"
|
|
fi
|
|
|
|
log_step "Step 9: Check Final State"
|
|
AGENTS_RESPONSE=$(ws_request "agent.list" '{}')
|
|
FINAL_AGENTS=$(echo "$AGENTS_RESPONSE" | jq '.payload.total')
|
|
log_info "Total agents: $FINAL_AGENTS"
|
|
|
|
TASK_RESPONSE=$(ws_request "task.get" "{\"id\": \"${TASK_ID}\"}")
|
|
TASK_STATE=$(echo "$TASK_RESPONSE" | jq -r '.payload.state')
|
|
log_info "Task state: $TASK_STATE"
|
|
|
|
log_step "Step 10: Verify First Prompt Output"
|
|
if [ -f "$AGENT_OUTPUT_FILE" ]; then
|
|
AGENT_OUTPUT=$(cat "$AGENT_OUTPUT_FILE")
|
|
log_success "Agent created file: $AGENT_OUTPUT_FILE"
|
|
log_info "File content: $AGENT_OUTPUT"
|
|
if echo "$AGENT_OUTPUT" | grep -qi "hello"; then
|
|
log_success "First prompt output contains expected content"
|
|
fi
|
|
else
|
|
log_error "Agent did not create expected file"
|
|
exit 1
|
|
fi
|
|
|
|
log_step "Step 11: Test Multi-Turn - Send Follow-up Prompt"
|
|
SECOND_OUTPUT_FILE="${TEST_WORKSPACE}/second-result.txt"
|
|
log_info "Sending follow-up prompt..."
|
|
PROMPT_PAYLOAD=$(cat <<EOF
|
|
{
|
|
"task_id": "${TASK_ID}",
|
|
"prompt": "Create another file named 'second-result.txt' with the content 'Follow-up successful'. Do nothing else."
|
|
}
|
|
EOF
|
|
)
|
|
PROMPT_RESPONSE=$(ws_request "orchestrator.prompt" "$PROMPT_PAYLOAD")
|
|
PROMPT_SUCCESS=$(echo "$PROMPT_RESPONSE" | jq -r '.payload.success // false')
|
|
if [ "$PROMPT_SUCCESS" == "true" ]; then
|
|
log_success "Follow-up prompt accepted"
|
|
else
|
|
log_error "Follow-up prompt failed: $(echo "$PROMPT_RESPONSE" | jq -r '.payload.error // "unknown"')"
|
|
exit 1
|
|
fi
|
|
|
|
# Wait for agent to become READY again (prompt completed)
|
|
log_info "Waiting for agent to become READY again..."
|
|
for i in $(seq 1 120); do
|
|
AGENT_STATUS_RESPONSE=$(ws_request "agent.status" "{\"agent_id\": \"${AGENT_ID}\"}" 2>/dev/null || echo '{}')
|
|
AGENT_STATUS=$(echo "$AGENT_STATUS_RESPONSE" | jq -r '.payload.status // "UNKNOWN"')
|
|
if [ "$AGENT_STATUS" == "READY" ]; then
|
|
log_success "Agent is READY after follow-up prompt ($i seconds)"
|
|
break
|
|
fi
|
|
if [ $((i % 10)) -eq 0 ]; then
|
|
log_info "Still waiting... ($i seconds, status: $AGENT_STATUS)"
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
# Verify second file was created
|
|
if [ -f "$SECOND_OUTPUT_FILE" ]; then
|
|
SECOND_OUTPUT=$(cat "$SECOND_OUTPUT_FILE")
|
|
log_success "Second file created: $SECOND_OUTPUT_FILE"
|
|
log_info "Second file content: $SECOND_OUTPUT"
|
|
else
|
|
log_error "Agent did not create second file"
|
|
exit 1
|
|
fi
|
|
|
|
log_step "Step 12: Complete Task"
|
|
COMPLETE_RESPONSE=$(ws_request "orchestrator.complete" "{\"task_id\": \"${TASK_ID}\"}")
|
|
COMPLETE_SUCCESS=$(echo "$COMPLETE_RESPONSE" | jq -r '.payload.success // false')
|
|
if [ "$COMPLETE_SUCCESS" == "true" ]; then
|
|
log_success "Task completed successfully"
|
|
else
|
|
log_error "Failed to complete task: $(echo "$COMPLETE_RESPONSE" | jq -r '.payload.error // "unknown"')"
|
|
fi
|
|
|
|
# Verify task state is COMPLETED
|
|
FINAL_TASK_RESPONSE=$(ws_request "task.get" "{\"id\": \"${TASK_ID}\"}")
|
|
TASK_STATE=$(echo "$FINAL_TASK_RESPONSE" | jq -r '.payload.state')
|
|
if [ "$TASK_STATE" == "COMPLETED" ]; then
|
|
log_success "Task state is COMPLETED"
|
|
else
|
|
log_error "Expected task state COMPLETED, got: $TASK_STATE"
|
|
fi
|
|
|
|
log_step "Step 13: Verify Session ID in Task Metadata"
|
|
# The session_id should have been stored in task metadata by the session_info handler
|
|
SESSION_ID=$(echo "$FINAL_TASK_RESPONSE" | jq -r '.payload.metadata.auggie_session_id // empty')
|
|
if [ -n "$SESSION_ID" ]; then
|
|
log_success "Session ID stored in task metadata: $SESSION_ID"
|
|
else
|
|
log_error "Session ID not found in task metadata"
|
|
log_info "Task metadata: $(echo "$FINAL_TASK_RESPONSE" | jq '.payload.metadata')"
|
|
exit 1
|
|
fi
|
|
|
|
# Track historical logs test results
|
|
HIST_LOGS_TEST_PASSED=false
|
|
HIST_LOGS_COUNT=0
|
|
HIST_SUBSCRIBE_TEST_PASSED=false
|
|
HIST_SUBSCRIBE_COUNT=0
|
|
LIVE_STREAM_TEST_PASSED=false
|
|
LIVE_STREAM_COUNT=0
|
|
|
|
log_step "Step 14: Test Historical Logs for Completed Task"
|
|
# Test task.logs action to retrieve all historical logs
|
|
LOGS_RESPONSE=$(ws_request "task.logs" "{\"task_id\": \"${TASK_ID}\"}")
|
|
LOGS_SUCCESS=$(echo "$LOGS_RESPONSE" | jq -e '.payload.logs' > /dev/null 2>&1 && echo "true" || echo "false")
|
|
|
|
if [ "$LOGS_SUCCESS" == "true" ]; then
|
|
HIST_LOGS_COUNT=$(echo "$LOGS_RESPONSE" | jq '.payload.logs | length')
|
|
TOTAL_FROM_RESPONSE=$(echo "$LOGS_RESPONSE" | jq '.payload.total // 0')
|
|
log_success "Retrieved $HIST_LOGS_COUNT historical logs (total: $TOTAL_FROM_RESPONSE)"
|
|
|
|
# Verify logs contain expected fields
|
|
if [ "$HIST_LOGS_COUNT" -gt 0 ]; then
|
|
FIRST_LOG=$(echo "$LOGS_RESPONSE" | jq '.payload.logs[0]')
|
|
HAS_TYPE=$(echo "$FIRST_LOG" | jq -e '.type' > /dev/null 2>&1 && echo "true" || echo "false")
|
|
HAS_TIMESTAMP=$(echo "$FIRST_LOG" | jq -e '.timestamp' > /dev/null 2>&1 && echo "true" || echo "false")
|
|
|
|
if [ "$HAS_TYPE" == "true" ] && [ "$HAS_TIMESTAMP" == "true" ]; then
|
|
log_success "Logs contain expected fields (type, timestamp)"
|
|
else
|
|
log_info "Log entry missing expected fields: $FIRST_LOG"
|
|
fi
|
|
|
|
# Check for ACP message types (session_info should exist)
|
|
SESSION_INFO_COUNT=$(echo "$LOGS_RESPONSE" | jq '[.payload.logs[] | select(.type == "session_info")] | length')
|
|
if [ "$SESSION_INFO_COUNT" -gt 0 ]; then
|
|
log_success "Found $SESSION_INFO_COUNT session_info log entries"
|
|
HIST_LOGS_TEST_PASSED=true
|
|
else
|
|
log_info "No session_info log entries found"
|
|
# Still consider it a pass if we got logs
|
|
HIST_LOGS_TEST_PASSED=true
|
|
fi
|
|
else
|
|
log_info "No historical logs found (empty array)"
|
|
fi
|
|
else
|
|
log_info "Failed to retrieve historical logs: $(echo "$LOGS_RESPONSE" | jq -r '.payload.error // "unknown error"')"
|
|
fi
|
|
|
|
log_step "Step 15: Test Historical Logs Replay on Subscribe"
|
|
# Subscribe to completed task and check if historical logs are replayed as notifications
|
|
# Use a temp file to collect messages
|
|
SUBSCRIBE_OUTPUT_FILE=$(mktemp)
|
|
|
|
# Send subscribe request and collect responses for a few seconds
|
|
(
|
|
echo '{"id":"hist-sub-1","type":"request","action":"task.subscribe","payload":{"task_id":"'"$TASK_ID"'"}}'
|
|
sleep 3
|
|
) | timeout 5 websocat "${WS_URL}" > "$SUBSCRIBE_OUTPUT_FILE" 2>/dev/null || true
|
|
|
|
# Count the messages received
|
|
if [ -f "$SUBSCRIBE_OUTPUT_FILE" ] && [ -s "$SUBSCRIBE_OUTPUT_FILE" ]; then
|
|
TOTAL_MESSAGES=$(wc -l < "$SUBSCRIBE_OUTPUT_FILE" | tr -d ' ')
|
|
log_info "Received $TOTAL_MESSAGES messages from subscribe"
|
|
|
|
# Check for the subscribe response
|
|
SUBSCRIBE_RESPONSE=$(grep -m1 '"action":"task.subscribe"' "$SUBSCRIBE_OUTPUT_FILE" || true)
|
|
if [ -n "$SUBSCRIBE_RESPONSE" ]; then
|
|
log_success "Received subscribe response"
|
|
fi
|
|
|
|
# Count historical notifications (acp.* actions)
|
|
ACP_MESSAGES=$(grep -c '"action":"acp\.' "$SUBSCRIBE_OUTPUT_FILE" 2>/dev/null || echo "0")
|
|
if [ "$ACP_MESSAGES" -gt 0 ]; then
|
|
log_success "Received $ACP_MESSAGES historical ACP notifications"
|
|
HIST_SUBSCRIBE_COUNT=$ACP_MESSAGES
|
|
HIST_SUBSCRIBE_TEST_PASSED=true
|
|
else
|
|
log_info "No historical ACP notifications received (may be expected if task has no ACP logs)"
|
|
# Check for any notification type messages
|
|
NOTIF_COUNT=$(grep -c '"type":"notification"' "$SUBSCRIBE_OUTPUT_FILE" 2>/dev/null || echo "0")
|
|
if [ "$NOTIF_COUNT" -gt 0 ]; then
|
|
log_info "Received $NOTIF_COUNT notification messages"
|
|
HIST_SUBSCRIBE_COUNT=$NOTIF_COUNT
|
|
HIST_SUBSCRIBE_TEST_PASSED=true
|
|
fi
|
|
fi
|
|
else
|
|
log_info "No messages received from subscribe (file empty or missing)"
|
|
fi
|
|
rm -f "$SUBSCRIBE_OUTPUT_FILE"
|
|
|
|
log_step "Step 16: Create and Monitor New Task with Live Streaming"
|
|
# Create a new task to test real-time streaming + historical logs
|
|
NEW_TEST_WORKSPACE=$(mktemp -d)
|
|
log_info "New test workspace: $NEW_TEST_WORKSPACE"
|
|
|
|
NEW_TASK_PAYLOAD=$(cat <<EOF
|
|
{
|
|
"title": "E2E Live Stream Test",
|
|
"description": "Create a file named 'stream-test.txt' with content 'Live streaming works'. Do nothing else.",
|
|
"board_id": "${BOARD_ID}",
|
|
"column_id": "${COLUMN_ID}",
|
|
"repository_url": "${NEW_TEST_WORKSPACE}",
|
|
"agent_type": "augment-agent"
|
|
}
|
|
EOF
|
|
)
|
|
NEW_TASK_RESPONSE=$(ws_request "task.create" "$NEW_TASK_PAYLOAD")
|
|
NEW_TASK_ID=$(echo "$NEW_TASK_RESPONSE" | jq -r '.payload.id')
|
|
|
|
if [ "$NEW_TASK_ID" == "null" ] || [ -z "$NEW_TASK_ID" ]; then
|
|
log_info "Failed to create new task for streaming test, skipping..."
|
|
else
|
|
log_success "Created new task: $NEW_TASK_ID"
|
|
|
|
# Subscribe FIRST before starting the task
|
|
STREAM_OUTPUT_FILE=$(mktemp)
|
|
log_info "Subscribing to task before starting..."
|
|
|
|
# Start collecting messages in background (subscribe, wait for start, collect for duration)
|
|
(
|
|
# Send subscribe request
|
|
echo '{"id":"live-sub-1","type":"request","action":"task.subscribe","payload":{"task_id":"'"$NEW_TASK_ID"'"}}'
|
|
# Keep connection open for streaming
|
|
sleep 90
|
|
) | timeout 95 websocat "${WS_URL}" > "$STREAM_OUTPUT_FILE" 2>/dev/null &
|
|
STREAM_PID=$!
|
|
|
|
# Wait a moment for subscription to be established
|
|
sleep 2
|
|
|
|
# Start the task
|
|
log_info "Starting task via orchestrator..."
|
|
START_RESP=$(ws_request "orchestrator.start" "{\"task_id\": \"${NEW_TASK_ID}\"}")
|
|
NEW_AGENT_ID=$(echo "$START_RESP" | jq -r '.payload.agent_instance_id')
|
|
|
|
if [ "$NEW_AGENT_ID" != "null" ] && [ -n "$NEW_AGENT_ID" ]; then
|
|
log_success "Task started, agent ID: $NEW_AGENT_ID"
|
|
|
|
# Wait for agent to complete (become READY)
|
|
log_info "Waiting for agent to complete..."
|
|
for i in $(seq 1 120); do
|
|
STATUS_RESP=$(ws_request "agent.status" "{\"agent_id\": \"${NEW_AGENT_ID}\"}" 2>/dev/null || echo '{}')
|
|
STATUS=$(echo "$STATUS_RESP" | jq -r '.payload.status // "UNKNOWN"')
|
|
if [ "$STATUS" == "READY" ]; then
|
|
log_success "Agent completed after $i seconds"
|
|
break
|
|
elif [ "$STATUS" == "FAILED" ]; then
|
|
log_info "Agent failed"
|
|
break
|
|
fi
|
|
if [ $((i % 15)) -eq 0 ]; then
|
|
log_info "Still waiting... ($i seconds, status: $STATUS)"
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
# Give a moment for final messages to arrive
|
|
sleep 2
|
|
|
|
# Kill the streaming process
|
|
kill $STREAM_PID 2>/dev/null || true
|
|
wait $STREAM_PID 2>/dev/null || true
|
|
|
|
# Count real-time messages received
|
|
if [ -f "$STREAM_OUTPUT_FILE" ] && [ -s "$STREAM_OUTPUT_FILE" ]; then
|
|
REALTIME_TOTAL=$(wc -l < "$STREAM_OUTPUT_FILE" | tr -d ' ')
|
|
REALTIME_NOTIF=$(grep -c '"type":"notification"' "$STREAM_OUTPUT_FILE" 2>/dev/null || echo "0")
|
|
REALTIME_ACP=$(grep -c '"action":"acp\.' "$STREAM_OUTPUT_FILE" 2>/dev/null || echo "0")
|
|
log_success "Real-time streaming: $REALTIME_TOTAL messages, $REALTIME_NOTIF notifications, $REALTIME_ACP ACP messages"
|
|
LIVE_STREAM_COUNT=$REALTIME_NOTIF
|
|
|
|
if [ "$REALTIME_NOTIF" -gt 0 ]; then
|
|
LIVE_STREAM_TEST_PASSED=true
|
|
fi
|
|
else
|
|
log_info "No streaming messages collected"
|
|
fi
|
|
|
|
# Complete the new task
|
|
ws_request "orchestrator.complete" "{\"task_id\": \"${NEW_TASK_ID}\"}" > /dev/null 2>&1 || true
|
|
|
|
# Now use task.logs to get full history
|
|
log_info "Retrieving full history via task.logs..."
|
|
NEW_LOGS_RESP=$(ws_request "task.logs" "{\"task_id\": \"${NEW_TASK_ID}\"}")
|
|
NEW_LOGS_COUNT=$(echo "$NEW_LOGS_RESP" | jq '.payload.logs | length' 2>/dev/null || echo "0")
|
|
|
|
if [ "$NEW_LOGS_COUNT" -gt 0 ]; then
|
|
log_success "task.logs returned $NEW_LOGS_COUNT historical entries"
|
|
|
|
# Compare with real-time count
|
|
if [ "$REALTIME_ACP" -gt 0 ]; then
|
|
log_info "Comparison: $REALTIME_ACP real-time ACP messages vs $NEW_LOGS_COUNT historical logs"
|
|
fi
|
|
else
|
|
log_info "task.logs returned no entries"
|
|
fi
|
|
else
|
|
log_info "Failed to start new task, skipping streaming test"
|
|
kill $STREAM_PID 2>/dev/null || true
|
|
fi
|
|
|
|
rm -f "$STREAM_OUTPUT_FILE"
|
|
rm -rf "$NEW_TEST_WORKSPACE" 2>/dev/null || true
|
|
fi
|
|
|
|
# Clean up test workspace
|
|
rm -rf "$TEST_WORKSPACE" 2>/dev/null || true
|
|
|
|
# Show backend logs summary
|
|
log_step "Backend Log Summary"
|
|
if [ -f /tmp/kandev-e2e.log ]; then
|
|
log_info "Key events from backend:"
|
|
grep -E "(agent|agentctl|container|task)" /tmp/kandev-e2e.log | tail -20 || true
|
|
fi
|
|
|
|
# Summary
|
|
log_step "Test Summary"
|
|
|
|
# Format historical logs test result
|
|
if [ "$HIST_LOGS_TEST_PASSED" == "true" ]; then
|
|
HIST_LOGS_RESULT="${GREEN}✓ ($HIST_LOGS_COUNT logs)${NC}"
|
|
else
|
|
HIST_LOGS_RESULT="${YELLOW}○ (skipped)${NC}"
|
|
fi
|
|
|
|
# Format subscribe replay test result
|
|
if [ "$HIST_SUBSCRIBE_TEST_PASSED" == "true" ]; then
|
|
HIST_SUBSCRIBE_RESULT="${GREEN}✓ ($HIST_SUBSCRIBE_COUNT msgs)${NC}"
|
|
else
|
|
HIST_SUBSCRIBE_RESULT="${YELLOW}○ (skipped)${NC}"
|
|
fi
|
|
|
|
# Format live streaming test result
|
|
if [ "$LIVE_STREAM_TEST_PASSED" == "true" ]; then
|
|
LIVE_STREAM_RESULT="${GREEN}✓ ($LIVE_STREAM_COUNT msgs)${NC}"
|
|
else
|
|
LIVE_STREAM_RESULT="${YELLOW}○ (skipped)${NC}"
|
|
fi
|
|
|
|
echo -e "${GREEN}╔═══════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${GREEN}║ End-to-End Test PASSED! ✓ ║${NC}"
|
|
echo -e "${GREEN}╠═══════════════════════════════════════════════════╣${NC}"
|
|
echo -e "${GREEN}║ Board created: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ Column created: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ Task created: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ Agent launched: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ agentctl running: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ First prompt: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ Multi-turn: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ Task completed: ✓ ║${NC}"
|
|
echo -e "${GREEN}║ Session ID stored: ✓ ║${NC}"
|
|
echo -e "${GREEN}╠═══════════════════════════════════════════════════╣${NC}"
|
|
echo -e "║ Historical Logs: $HIST_LOGS_RESULT"
|
|
echo -e "║ Subscribe Replay: $HIST_SUBSCRIBE_RESULT"
|
|
echo -e "║ Live Streaming: $LIVE_STREAM_RESULT"
|
|
echo -e "${GREEN}╚═══════════════════════════════════════════════════╝${NC}"
|
|
|
|
exit 0
|
|
|