first commit
This commit is contained in:
47
.env.example
Normal file
47
.env.example
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
# ============================================
|
||||||
|
# OSINT Platform Environment Configuration
|
||||||
|
# ============================================
|
||||||
|
# Copy this file to .env and configure your values
|
||||||
|
|
||||||
|
# Server Configuration
|
||||||
|
PORT=3001
|
||||||
|
NODE_ENV=production
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# DATABASE (MongoDB)
|
||||||
|
# ===========================================
|
||||||
|
|
||||||
|
# MongoDB Connection URI
|
||||||
|
# For local: mongodb://localhost:27017/osint_platform
|
||||||
|
# For Atlas: mongodb+srv://user:password@cluster.mongodb.net/osint_platform
|
||||||
|
MONGODB_URI=mongodb://localhost:27017/osint_platform
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# REQUIRED SECURITY SETTINGS
|
||||||
|
# ===========================================
|
||||||
|
|
||||||
|
# Master Password - CHANGE THIS!
|
||||||
|
# This is the only password protecting your entire platform
|
||||||
|
MASTER_PASSWORD=your_secure_master_password_here
|
||||||
|
|
||||||
|
# JWT Secret - Generate a random 64-character string
|
||||||
|
# You can use: openssl rand -hex 32
|
||||||
|
JWT_SECRET=your_jwt_secret_here_generate_with_openssl_rand_hex_32
|
||||||
|
|
||||||
|
# Session Configuration
|
||||||
|
SESSION_EXPIRY=24h
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# ENCRYPTION SETTINGS
|
||||||
|
# ===========================================
|
||||||
|
|
||||||
|
# Vault Encryption Key - Generate a 64-character hex string (32 bytes)
|
||||||
|
# You can use: openssl rand -hex 32
|
||||||
|
VAULT_ENCRYPTION_KEY=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
||||||
|
|
||||||
|
# ===========================================
|
||||||
|
# FRONTEND URL (for CORS)
|
||||||
|
# ===========================================
|
||||||
|
|
||||||
|
# Set this to your actual domain in production
|
||||||
|
FRONTEND_URL=http://localhost:3001
|
||||||
44
.gitignore
vendored
Normal file
44
.gitignore
vendored
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
# Dependencies
|
||||||
|
node_modules/
|
||||||
|
|
||||||
|
# Build outputs
|
||||||
|
dist/
|
||||||
|
frontend/dist/
|
||||||
|
backend/dist/
|
||||||
|
|
||||||
|
# Environment files
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
.env.*.local
|
||||||
|
|
||||||
|
# Database
|
||||||
|
*.db
|
||||||
|
*.db-journal
|
||||||
|
*.db-wal
|
||||||
|
*.db-shm
|
||||||
|
data/
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
logs/
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Test coverage
|
||||||
|
coverage/
|
||||||
|
|
||||||
|
# Playwright
|
||||||
|
playwright-report/
|
||||||
|
test-results/
|
||||||
|
|
||||||
|
# Misc
|
||||||
|
*.pem
|
||||||
|
*.key
|
||||||
111
Dockerfile
Normal file
111
Dockerfile
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
# ============================================
|
||||||
|
# OSINT Platform - Multi-stage Dockerfile
|
||||||
|
# Optimized for Coolify deployment
|
||||||
|
# ============================================
|
||||||
|
|
||||||
|
# Stage 1: Build frontend
|
||||||
|
FROM node:20-alpine AS frontend-builder
|
||||||
|
|
||||||
|
WORKDIR /app/frontend
|
||||||
|
|
||||||
|
# Copy frontend package files
|
||||||
|
COPY frontend/package*.json ./
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
RUN npm ci --legacy-peer-deps
|
||||||
|
|
||||||
|
# Copy frontend source
|
||||||
|
COPY frontend/ ./
|
||||||
|
|
||||||
|
# Build frontend
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
# ============================================
|
||||||
|
# Stage 2: Build backend
|
||||||
|
FROM node:20-alpine AS backend-builder
|
||||||
|
|
||||||
|
WORKDIR /app/backend
|
||||||
|
|
||||||
|
# Copy backend package files
|
||||||
|
COPY backend/package*.json ./
|
||||||
|
|
||||||
|
# Install dependencies (including dev deps for build)
|
||||||
|
RUN npm ci
|
||||||
|
|
||||||
|
# Copy backend source
|
||||||
|
COPY backend/ ./
|
||||||
|
|
||||||
|
# Build TypeScript
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
# ============================================
|
||||||
|
# Stage 3: Production runtime
|
||||||
|
FROM node:20-slim AS production
|
||||||
|
|
||||||
|
# Install Playwright dependencies
|
||||||
|
# This includes all browser deps for Chromium
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
# Playwright/Chromium dependencies
|
||||||
|
libnss3 \
|
||||||
|
libnspr4 \
|
||||||
|
libatk1.0-0 \
|
||||||
|
libatk-bridge2.0-0 \
|
||||||
|
libcups2 \
|
||||||
|
libdrm2 \
|
||||||
|
libdbus-1-3 \
|
||||||
|
libxkbcommon0 \
|
||||||
|
libxcomposite1 \
|
||||||
|
libxdamage1 \
|
||||||
|
libxfixes3 \
|
||||||
|
libxrandr2 \
|
||||||
|
libgbm1 \
|
||||||
|
libasound2 \
|
||||||
|
libpango-1.0-0 \
|
||||||
|
libcairo2 \
|
||||||
|
libatspi2.0-0 \
|
||||||
|
# Additional utilities
|
||||||
|
wget \
|
||||||
|
ca-certificates \
|
||||||
|
fonts-liberation \
|
||||||
|
# Clean up
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& apt-get clean
|
||||||
|
|
||||||
|
# Create non-root user for security
|
||||||
|
RUN groupadd -r osint && useradd -r -g osint osint
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy backend production files
|
||||||
|
COPY --from=backend-builder /app/backend/dist ./dist
|
||||||
|
COPY --from=backend-builder /app/backend/package*.json ./
|
||||||
|
|
||||||
|
# Install production dependencies only
|
||||||
|
RUN npm ci --only=production --legacy-peer-deps
|
||||||
|
|
||||||
|
# Install Playwright browsers (Chromium only for smaller image)
|
||||||
|
RUN npx playwright install chromium
|
||||||
|
|
||||||
|
# Copy frontend build
|
||||||
|
COPY --from=frontend-builder /app/frontend/dist ./frontend/dist
|
||||||
|
|
||||||
|
# Create logs directory with proper permissions
|
||||||
|
RUN mkdir -p /app/logs && \
|
||||||
|
chown -R osint:osint /app
|
||||||
|
|
||||||
|
# Environment variables
|
||||||
|
ENV NODE_ENV=production
|
||||||
|
ENV PORT=3001
|
||||||
|
|
||||||
|
# Expose port
|
||||||
|
EXPOSE 3001
|
||||||
|
|
||||||
|
# Health check
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
|
||||||
|
CMD wget --no-verbose --tries=1 --spider http://localhost:3001/api/health || exit 1
|
||||||
|
|
||||||
|
# Switch to non-root user
|
||||||
|
USER osint
|
||||||
|
|
||||||
|
# Start the application
|
||||||
|
CMD ["node", "dist/index.js"]
|
||||||
182
README.md
Normal file
182
README.md
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
# OSINT Automation Platform
|
||||||
|
|
||||||
|
A private, containerized OSINT (Open Source Intelligence) automation platform for gathering social media intelligence using authenticated browser sessions.
|
||||||
|
|
||||||
|
⚠️ **DISCLAIMER**: This tool is intended for authorized intelligence gathering and research purposes only. Ensure you comply with all applicable laws and platform terms of service.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- 🔐 **Master Password Protection** - Single secure entry point
|
||||||
|
- 🎭 **Session Vault** - Store and reuse authenticated social media sessions (encrypted)
|
||||||
|
- 🤖 **Automated Scraping** - Playwright-based stealth scraping engine
|
||||||
|
- 📊 **Target Management** - Organize and track multiple investigation targets
|
||||||
|
- 📱 **Multi-Platform Support** - X/Twitter, Instagram, LinkedIn, Facebook
|
||||||
|
- 🔒 **Encrypted Storage** - AES-256-GCM encrypted session data
|
||||||
|
- 📈 **Real-time Progress** - WebSocket-based live job updates
|
||||||
|
- 🐳 **Docker Ready** - Optimized for Coolify deployment
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
|
||||||
|
- **Backend**: Node.js, Express, TypeScript
|
||||||
|
- **Frontend**: React, Vite, TypeScript, Tailwind CSS
|
||||||
|
- **Database**: MongoDB
|
||||||
|
- **Browser Automation**: Playwright with stealth plugins
|
||||||
|
- **Real-time**: Socket.IO
|
||||||
|
- **State Management**: Zustand
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Development
|
||||||
|
|
||||||
|
1. Clone the repository:
|
||||||
|
```bash
|
||||||
|
git clone <repo-url>
|
||||||
|
cd osint-platform
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Install dependencies:
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
cd backend && npm install
|
||||||
|
cd ../frontend && npm install
|
||||||
|
cd ..
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Create environment files:
|
||||||
|
```bash
|
||||||
|
# Backend
|
||||||
|
cp backend/.env.example backend/.env
|
||||||
|
# Edit backend/.env with your configuration (especially MONGODB_URI)
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Start development servers:
|
||||||
|
```bash
|
||||||
|
npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
This will start:
|
||||||
|
- Backend on `http://localhost:3001`
|
||||||
|
- Frontend on `http://localhost:5173`
|
||||||
|
|
||||||
|
### Production (Docker)
|
||||||
|
|
||||||
|
1. Create your `.env` file:
|
||||||
|
```bash
|
||||||
|
cp .env.example .env
|
||||||
|
# Edit .env with secure values
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Build and run with Docker Compose:
|
||||||
|
```bash
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
The application will be available at `http://localhost:3001`
|
||||||
|
|
||||||
|
## Coolify Deployment
|
||||||
|
|
||||||
|
1. Create a new service in Coolify
|
||||||
|
2. Point to your Git repository
|
||||||
|
3. Set Build Pack to "Dockerfile"
|
||||||
|
4. Configure environment variables:
|
||||||
|
- `MONGODB_URI` - Your MongoDB connection string (e.g., MongoDB Atlas)
|
||||||
|
- `MASTER_PASSWORD` - Your secure master password
|
||||||
|
- `JWT_SECRET` - Generate with `openssl rand -hex 32`
|
||||||
|
- `VAULT_ENCRYPTION_KEY` - Generate with `openssl rand -hex 32`
|
||||||
|
- `FRONTEND_URL` - Your domain (e.g., `https://osint.yourdomain.com`)
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
| Variable | Description | Required |
|
||||||
|
|----------|-------------|----------|
|
||||||
|
| `MONGODB_URI` | MongoDB connection string | Yes |
|
||||||
|
| `MASTER_PASSWORD` | Password to access the platform | Yes |
|
||||||
|
| `JWT_SECRET` | Secret for JWT token signing | Yes |
|
||||||
|
| `VAULT_ENCRYPTION_KEY` | 64-char hex key for session encryption | Yes |
|
||||||
|
| `PORT` | Server port (default: 3001) | No |
|
||||||
|
| `SESSION_EXPIRY` | JWT expiry time (default: 24h) | No |
|
||||||
|
| `FRONTEND_URL` | Frontend URL for CORS | No |
|
||||||
|
|
||||||
|
### Generating Secrets
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Generate JWT Secret
|
||||||
|
openssl rand -hex 32
|
||||||
|
|
||||||
|
# Generate Vault Encryption Key
|
||||||
|
openssl rand -hex 32
|
||||||
|
```
|
||||||
|
|
||||||
|
### MongoDB Setup
|
||||||
|
|
||||||
|
You can use:
|
||||||
|
- **MongoDB Atlas** (recommended for production): Create a free cluster at [mongodb.com/atlas](https://mongodb.com/atlas)
|
||||||
|
- **Local MongoDB**: `mongodb://localhost:27017/osint_platform`
|
||||||
|
- **Docker MongoDB**: Uncomment the mongodb service in docker-compose.yml
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### 1. Login
|
||||||
|
Access the platform and enter your master password.
|
||||||
|
|
||||||
|
### 2. Add Sessions
|
||||||
|
Navigate to "Add Session" and provide:
|
||||||
|
- Platform (X, Instagram, LinkedIn, Facebook)
|
||||||
|
- Session name
|
||||||
|
- Cookies JSON (export from your browser)
|
||||||
|
|
||||||
|
**Getting cookies:**
|
||||||
|
1. Log into the platform in your browser
|
||||||
|
2. Open DevTools → Application → Cookies
|
||||||
|
3. Export as JSON using the Cookie Editor extension
|
||||||
|
|
||||||
|
### 3. Create Targets
|
||||||
|
Add investigation targets with optional notes.
|
||||||
|
|
||||||
|
### 4. Add Profiles
|
||||||
|
Link social media profiles to targets:
|
||||||
|
- Platform
|
||||||
|
- Username
|
||||||
|
- Profile URL
|
||||||
|
|
||||||
|
### 5. Run Scrapers
|
||||||
|
Click the play button on any profile to start scraping. Monitor progress in the right panel.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
osint-platform/
|
||||||
|
├── backend/
|
||||||
|
│ ├── src/
|
||||||
|
│ │ ├── database/ # MongoDB connection
|
||||||
|
│ │ ├── middleware/ # Auth middleware
|
||||||
|
│ │ ├── models/ # Mongoose models
|
||||||
|
│ │ ├── routes/ # API routes
|
||||||
|
│ │ ├── scraper/ # Playwright scraper engine
|
||||||
|
│ │ └── utils/ # Encryption, logging
|
||||||
|
│ └── package.json
|
||||||
|
├── frontend/
|
||||||
|
│ ├── src/
|
||||||
|
│ │ ├── components/ # React components
|
||||||
|
│ │ ├── pages/ # Page components
|
||||||
|
│ │ └── stores/ # Zustand stores
|
||||||
|
│ └── package.json
|
||||||
|
├── Dockerfile
|
||||||
|
├── docker-compose.yml
|
||||||
|
└── package.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
1. **Use strong passwords** - The master password is your only line of defense
|
||||||
|
2. **Secure your secrets** - Never commit `.env` files
|
||||||
|
3. **Use HTTPS** - Always deploy behind HTTPS in production
|
||||||
|
4. **Rate limiting** - Login attempts are rate-limited (5 per 15 minutes)
|
||||||
|
5. **Session encryption** - All stored cookies are AES-256-GCM encrypted
|
||||||
|
6. **MongoDB Security** - Use authentication and TLS for your MongoDB connection
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
Private - All rights reserved.
|
||||||
3114
backend/package-lock.json
generated
Normal file
3114
backend/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
40
backend/package.json
Normal file
40
backend/package.json
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
{
|
||||||
|
"name": "osint-backend",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "OSINT Platform Backend",
|
||||||
|
"type": "module",
|
||||||
|
"main": "dist/index.js",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "tsx watch src/index.ts",
|
||||||
|
"build": "tsc",
|
||||||
|
"start": "node dist/index.js"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"express": "^4.18.2",
|
||||||
|
"cors": "^2.8.5",
|
||||||
|
"helmet": "^7.1.0",
|
||||||
|
"bcryptjs": "^2.4.3",
|
||||||
|
"jsonwebtoken": "^9.0.2",
|
||||||
|
"cookie-parser": "^1.4.6",
|
||||||
|
"mongoose": "^8.2.1",
|
||||||
|
"playwright": "^1.42.1",
|
||||||
|
"playwright-extra": "^4.3.6",
|
||||||
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||||
|
"dotenv": "^16.4.5",
|
||||||
|
"uuid": "^9.0.1",
|
||||||
|
"winston": "^3.11.0",
|
||||||
|
"express-rate-limit": "^7.2.0",
|
||||||
|
"socket.io": "^4.7.5"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/express": "^4.17.21",
|
||||||
|
"@types/cors": "^2.8.17",
|
||||||
|
"@types/bcryptjs": "^2.4.6",
|
||||||
|
"@types/jsonwebtoken": "^9.0.6",
|
||||||
|
"@types/cookie-parser": "^1.4.7",
|
||||||
|
"@types/uuid": "^9.0.8",
|
||||||
|
"@types/node": "^20.11.24",
|
||||||
|
"typescript": "^5.4.2",
|
||||||
|
"tsx": "^4.7.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
35
backend/src/database/index.ts
Normal file
35
backend/src/database/index.ts
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { logger } from '../utils/logger.js';
|
||||||
|
|
||||||
|
export async function connectDatabase(): Promise<void> {
|
||||||
|
const mongoUri = process.env.MONGODB_URI;
|
||||||
|
|
||||||
|
if (!mongoUri) {
|
||||||
|
throw new Error('MONGODB_URI environment variable is required');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await mongoose.connect(mongoUri, {
|
||||||
|
dbName: 'osint_platform',
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info('Connected to MongoDB');
|
||||||
|
|
||||||
|
mongoose.connection.on('error', (err) => {
|
||||||
|
logger.error('MongoDB connection error:', err);
|
||||||
|
});
|
||||||
|
|
||||||
|
mongoose.connection.on('disconnected', () => {
|
||||||
|
logger.warn('MongoDB disconnected');
|
||||||
|
});
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to connect to MongoDB:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function closeDatabase(): Promise<void> {
|
||||||
|
await mongoose.connection.close();
|
||||||
|
logger.info('MongoDB connection closed');
|
||||||
|
}
|
||||||
131
backend/src/index.ts
Normal file
131
backend/src/index.ts
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
import express from 'express';
|
||||||
|
import cors from 'cors';
|
||||||
|
import helmet from 'helmet';
|
||||||
|
import cookieParser from 'cookie-parser';
|
||||||
|
import { createServer } from 'http';
|
||||||
|
import { Server } from 'socket.io';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
import path from 'path';
|
||||||
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
|
import { authRouter } from './routes/auth.js';
|
||||||
|
import { targetsRouter } from './routes/targets.js';
|
||||||
|
import { sessionsRouter } from './routes/sessions.js';
|
||||||
|
import { scraperRouter } from './routes/scraper.js';
|
||||||
|
import { authMiddleware } from './middleware/auth.js';
|
||||||
|
import { connectDatabase } from './database/index.js';
|
||||||
|
import { logger } from './utils/logger.js';
|
||||||
|
import { ScraperManager } from './scraper/manager.js';
|
||||||
|
|
||||||
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
|
const __dirname = path.dirname(__filename);
|
||||||
|
|
||||||
|
dotenv.config({ path: path.join(__dirname, '../../.env') });
|
||||||
|
|
||||||
|
const app = express();
|
||||||
|
const httpServer = createServer(app);
|
||||||
|
const io = new Server(httpServer, {
|
||||||
|
cors: {
|
||||||
|
origin: process.env.FRONTEND_URL || 'http://localhost:5173',
|
||||||
|
credentials: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const PORT = process.env.PORT || 3001;
|
||||||
|
|
||||||
|
// Initialize scraper manager with socket.io
|
||||||
|
const scraperManager = new ScraperManager(io);
|
||||||
|
app.set('scraperManager', scraperManager);
|
||||||
|
app.set('io', io);
|
||||||
|
|
||||||
|
// Middleware
|
||||||
|
app.use(helmet({
|
||||||
|
contentSecurityPolicy: false,
|
||||||
|
}));
|
||||||
|
app.use(cors({
|
||||||
|
origin: process.env.FRONTEND_URL || 'http://localhost:5173',
|
||||||
|
credentials: true,
|
||||||
|
}));
|
||||||
|
app.use(express.json());
|
||||||
|
app.use(cookieParser());
|
||||||
|
|
||||||
|
// Serve static files from frontend build in production
|
||||||
|
if (process.env.NODE_ENV === 'production') {
|
||||||
|
app.use(express.static(path.join(__dirname, '../../frontend/dist')));
|
||||||
|
}
|
||||||
|
|
||||||
|
// API Routes
|
||||||
|
app.use('/api/auth', authRouter);
|
||||||
|
app.use('/api/targets', authMiddleware, targetsRouter);
|
||||||
|
app.use('/api/sessions', authMiddleware, sessionsRouter);
|
||||||
|
app.use('/api/scraper', authMiddleware, scraperRouter);
|
||||||
|
|
||||||
|
// Health check
|
||||||
|
app.get('/api/health', (req, res) => {
|
||||||
|
res.json({ status: 'ok', timestamp: new Date().toISOString() });
|
||||||
|
});
|
||||||
|
|
||||||
|
// Catch-all for SPA routing in production
|
||||||
|
if (process.env.NODE_ENV === 'production') {
|
||||||
|
app.get('*', (req, res) => {
|
||||||
|
res.sendFile(path.join(__dirname, '../../frontend/dist/index.html'));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Socket.io authentication
|
||||||
|
io.use((socket, next) => {
|
||||||
|
const token = socket.handshake.auth.token;
|
||||||
|
if (!token) {
|
||||||
|
return next(new Error('Authentication required'));
|
||||||
|
}
|
||||||
|
// Token validation would go here
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
|
||||||
|
io.on('connection', (socket) => {
|
||||||
|
logger.info(`Client connected: ${socket.id}`);
|
||||||
|
|
||||||
|
socket.on('subscribe:scraper', (scraperId: string) => {
|
||||||
|
socket.join(`scraper:${scraperId}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('unsubscribe:scraper', (scraperId: string) => {
|
||||||
|
socket.leave(`scraper:${scraperId}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('disconnect', () => {
|
||||||
|
logger.info(`Client disconnected: ${socket.id}`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Global error handlers
|
||||||
|
process.on('unhandledRejection', (reason, promise) => {
|
||||||
|
logger.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||||
|
});
|
||||||
|
|
||||||
|
process.on('uncaughtException', (error) => {
|
||||||
|
logger.error('Uncaught Exception:', error);
|
||||||
|
// Important: give the logger time to write before exiting
|
||||||
|
setTimeout(() => {
|
||||||
|
process.exit(1);
|
||||||
|
}, 1000);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start server with database connection
|
||||||
|
async function start() {
|
||||||
|
try {
|
||||||
|
await connectDatabase();
|
||||||
|
|
||||||
|
httpServer.listen(PORT, () => {
|
||||||
|
logger.info(`🔒 OSINT Platform Backend running on port ${PORT}`);
|
||||||
|
logger.info(`📊 Environment: ${process.env.NODE_ENV || 'development'}`);
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to start server:', error);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
start();
|
||||||
|
|
||||||
|
export { app, io };
|
||||||
55
backend/src/middleware/auth.ts
Normal file
55
backend/src/middleware/auth.ts
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
import { Request, Response, NextFunction } from 'express';
|
||||||
|
import jwt from 'jsonwebtoken';
|
||||||
|
import { logger } from '../utils/logger.js';
|
||||||
|
|
||||||
|
export interface AuthRequest extends Request {
|
||||||
|
userId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function authMiddleware(req: AuthRequest, res: Response, next: NextFunction): void {
|
||||||
|
try {
|
||||||
|
// Check for token in Authorization header or cookie
|
||||||
|
const authHeader = req.headers.authorization;
|
||||||
|
const cookieToken = req.cookies?.auth_token;
|
||||||
|
|
||||||
|
let token: string | undefined;
|
||||||
|
|
||||||
|
if (authHeader && authHeader.startsWith('Bearer ')) {
|
||||||
|
token = authHeader.substring(7);
|
||||||
|
} else if (cookieToken) {
|
||||||
|
token = cookieToken;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!token) {
|
||||||
|
res.status(401).json({ error: 'Authentication required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const jwtSecret = process.env.JWT_SECRET;
|
||||||
|
if (!jwtSecret) {
|
||||||
|
logger.error('JWT_SECRET not configured');
|
||||||
|
res.status(500).json({ error: 'Server configuration error' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const decoded = jwt.verify(token, jwtSecret) as { authenticated: boolean };
|
||||||
|
|
||||||
|
if (!decoded.authenticated) {
|
||||||
|
res.status(401).json({ error: 'Invalid token' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
next();
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof jwt.TokenExpiredError) {
|
||||||
|
res.status(401).json({ error: 'Token expired' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (error instanceof jwt.JsonWebTokenError) {
|
||||||
|
res.status(401).json({ error: 'Invalid token' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.error('Auth middleware error:', error);
|
||||||
|
res.status(500).json({ error: 'Authentication error' });
|
||||||
|
}
|
||||||
|
}
|
||||||
52
backend/src/models/ScraperJob.ts
Normal file
52
backend/src/models/ScraperJob.ts
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
import mongoose, { Document, Schema } from 'mongoose';
|
||||||
|
|
||||||
|
export interface IScraperLog {
|
||||||
|
level: string;
|
||||||
|
message: string;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IScraperJob extends Document {
|
||||||
|
targetId?: mongoose.Types.ObjectId;
|
||||||
|
profileId?: mongoose.Types.ObjectId;
|
||||||
|
platform: string;
|
||||||
|
status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
|
||||||
|
progress: number;
|
||||||
|
result?: Record<string, any>;
|
||||||
|
error?: string;
|
||||||
|
logs: IScraperLog[];
|
||||||
|
startedAt?: Date;
|
||||||
|
completedAt?: Date;
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ScraperLogSchema = new Schema<IScraperLog>({
|
||||||
|
level: { type: String, required: true },
|
||||||
|
message: { type: String, required: true },
|
||||||
|
timestamp: { type: Date, default: Date.now },
|
||||||
|
}, { _id: false });
|
||||||
|
|
||||||
|
const ScraperJobSchema = new Schema<IScraperJob>({
|
||||||
|
targetId: { type: Schema.Types.ObjectId, ref: 'Target' },
|
||||||
|
profileId: { type: Schema.Types.ObjectId },
|
||||||
|
platform: { type: String, required: true },
|
||||||
|
status: {
|
||||||
|
type: String,
|
||||||
|
enum: ['pending', 'running', 'completed', 'failed', 'cancelled'],
|
||||||
|
default: 'pending'
|
||||||
|
},
|
||||||
|
progress: { type: Number, default: 0 },
|
||||||
|
result: { type: Schema.Types.Mixed },
|
||||||
|
error: { type: String },
|
||||||
|
logs: [ScraperLogSchema],
|
||||||
|
startedAt: { type: Date },
|
||||||
|
completedAt: { type: Date },
|
||||||
|
}, {
|
||||||
|
timestamps: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Index for efficient querying
|
||||||
|
ScraperJobSchema.index({ status: 1, createdAt: -1 });
|
||||||
|
ScraperJobSchema.index({ targetId: 1 });
|
||||||
|
|
||||||
|
export const ScraperJob = mongoose.model<IScraperJob>('ScraperJob', ScraperJobSchema);
|
||||||
33
backend/src/models/Session.ts
Normal file
33
backend/src/models/Session.ts
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
import mongoose, { Document, Schema } from 'mongoose';
|
||||||
|
|
||||||
|
export interface ISession extends Document {
|
||||||
|
platform: string;
|
||||||
|
sessionName: string;
|
||||||
|
cookiesEncrypted: string;
|
||||||
|
localStorageEncrypted?: string;
|
||||||
|
userAgent?: string;
|
||||||
|
proxy?: string;
|
||||||
|
status: 'active' | 'expired' | 'invalid';
|
||||||
|
lastValidated?: Date;
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
const SessionSchema = new Schema<ISession>({
|
||||||
|
platform: { type: String, required: true, unique: true },
|
||||||
|
sessionName: { type: String, required: true },
|
||||||
|
cookiesEncrypted: { type: String, required: true },
|
||||||
|
localStorageEncrypted: { type: String },
|
||||||
|
userAgent: { type: String },
|
||||||
|
proxy: { type: String },
|
||||||
|
status: {
|
||||||
|
type: String,
|
||||||
|
enum: ['active', 'expired', 'invalid'],
|
||||||
|
default: 'active'
|
||||||
|
},
|
||||||
|
lastValidated: { type: Date },
|
||||||
|
}, {
|
||||||
|
timestamps: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const Session = mongoose.model<ISession>('Session', SessionSchema);
|
||||||
47
backend/src/models/Target.ts
Normal file
47
backend/src/models/Target.ts
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
import mongoose, { Document, Schema } from 'mongoose';
|
||||||
|
|
||||||
|
export interface ISocialProfile {
|
||||||
|
_id: mongoose.Types.ObjectId;
|
||||||
|
platform: string;
|
||||||
|
username?: string;
|
||||||
|
profileUrl?: string;
|
||||||
|
profileData?: Record<string, any>;
|
||||||
|
lastScraped?: Date;
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ITarget extends Document {
|
||||||
|
name: string;
|
||||||
|
notes?: string;
|
||||||
|
profiles: ISocialProfile[];
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
const SocialProfileSchema = new Schema<ISocialProfile>({
|
||||||
|
platform: { type: String, required: true },
|
||||||
|
username: { type: String },
|
||||||
|
profileUrl: { type: String },
|
||||||
|
profileData: { type: Schema.Types.Mixed },
|
||||||
|
lastScraped: { type: Date },
|
||||||
|
createdAt: { type: Date, default: Date.now },
|
||||||
|
});
|
||||||
|
|
||||||
|
const TargetSchema = new Schema<ITarget>({
|
||||||
|
name: { type: String, required: true },
|
||||||
|
notes: { type: String },
|
||||||
|
profiles: [SocialProfileSchema],
|
||||||
|
}, {
|
||||||
|
timestamps: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Virtual for profile count
|
||||||
|
TargetSchema.virtual('profileCount').get(function() {
|
||||||
|
return this.profiles?.length || 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Ensure virtuals are serialized
|
||||||
|
TargetSchema.set('toJSON', { virtuals: true });
|
||||||
|
TargetSchema.set('toObject', { virtuals: true });
|
||||||
|
|
||||||
|
export const Target = mongoose.model<ITarget>('Target', TargetSchema);
|
||||||
3
backend/src/models/index.ts
Normal file
3
backend/src/models/index.ts
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
export { Target, type ITarget, type ISocialProfile } from './Target.js';
|
||||||
|
export { Session, type ISession } from './Session.js';
|
||||||
|
export { ScraperJob, type IScraperJob, type IScraperLog } from './ScraperJob.js';
|
||||||
113
backend/src/routes/auth.ts
Normal file
113
backend/src/routes/auth.ts
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
import { Router, Request, Response } from 'express';
|
||||||
|
import bcrypt from 'bcryptjs';
|
||||||
|
import jwt from 'jsonwebtoken';
|
||||||
|
import rateLimit from 'express-rate-limit';
|
||||||
|
import { logger } from '../utils/logger.js';
|
||||||
|
|
||||||
|
export const authRouter = Router();
|
||||||
|
|
||||||
|
// Rate limiting for auth endpoints
|
||||||
|
const authLimiter = rateLimit({
|
||||||
|
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||||
|
max: 5, // 5 attempts per window
|
||||||
|
message: { error: 'Too many authentication attempts, please try again later' },
|
||||||
|
standardHeaders: true,
|
||||||
|
legacyHeaders: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Login with master password
|
||||||
|
authRouter.post('/login', authLimiter, async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { password } = req.body;
|
||||||
|
|
||||||
|
if (!password) {
|
||||||
|
res.status(400).json({ error: 'Password is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const masterPassword = process.env.MASTER_PASSWORD;
|
||||||
|
if (!masterPassword) {
|
||||||
|
logger.error('MASTER_PASSWORD not configured');
|
||||||
|
res.status(500).json({ error: 'Server configuration error' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simple comparison for now - in production you'd hash the stored password
|
||||||
|
const isValid = password === masterPassword;
|
||||||
|
|
||||||
|
if (!isValid) {
|
||||||
|
logger.warn('Failed login attempt');
|
||||||
|
res.status(401).json({ error: 'Invalid password' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const jwtSecret = process.env.JWT_SECRET;
|
||||||
|
if (!jwtSecret) {
|
||||||
|
logger.error('JWT_SECRET not configured');
|
||||||
|
res.status(500).json({ error: 'Server configuration error' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = jwt.sign(
|
||||||
|
{ authenticated: true },
|
||||||
|
jwtSecret as string,
|
||||||
|
{ expiresIn: (process.env.SESSION_EXPIRY || '24h') as any }
|
||||||
|
);
|
||||||
|
|
||||||
|
// Set HTTP-only cookie
|
||||||
|
res.cookie('auth_token', token, {
|
||||||
|
httpOnly: true,
|
||||||
|
secure: process.env.NODE_ENV === 'production',
|
||||||
|
sameSite: 'strict',
|
||||||
|
maxAge: 24 * 60 * 60 * 1000, // 24 hours
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info('Successful login');
|
||||||
|
res.json({
|
||||||
|
success: true,
|
||||||
|
token,
|
||||||
|
expiresIn: process.env.SESSION_EXPIRY || '24h'
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Login error:', error);
|
||||||
|
res.status(500).json({ error: 'Authentication failed' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Verify token
|
||||||
|
authRouter.get('/verify', (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const authHeader = req.headers.authorization;
|
||||||
|
const cookieToken = req.cookies?.auth_token;
|
||||||
|
|
||||||
|
let token: string | undefined;
|
||||||
|
|
||||||
|
if (authHeader && authHeader.startsWith('Bearer ')) {
|
||||||
|
token = authHeader.substring(7);
|
||||||
|
} else if (cookieToken) {
|
||||||
|
token = cookieToken;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!token) {
|
||||||
|
res.status(401).json({ authenticated: false });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const jwtSecret = process.env.JWT_SECRET;
|
||||||
|
if (!jwtSecret) {
|
||||||
|
res.status(500).json({ error: 'Server configuration error' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
jwt.verify(token, jwtSecret);
|
||||||
|
res.json({ authenticated: true });
|
||||||
|
} catch (error) {
|
||||||
|
res.status(401).json({ authenticated: false });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Logout
|
||||||
|
authRouter.post('/logout', (req: Request, res: Response) => {
|
||||||
|
res.clearCookie('auth_token');
|
||||||
|
res.json({ success: true });
|
||||||
|
});
|
||||||
199
backend/src/routes/scraper.ts
Normal file
199
backend/src/routes/scraper.ts
Normal file
@@ -0,0 +1,199 @@
|
|||||||
|
import { Router, Request, Response } from 'express';
|
||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { ScraperJob } from '../models/ScraperJob.js';
|
||||||
|
import { Target } from '../models/Target.js';
|
||||||
|
import { logger } from '../utils/logger.js';
|
||||||
|
import { ScraperManager } from '../scraper/manager.js';
|
||||||
|
|
||||||
|
export const scraperRouter = Router();
|
||||||
|
|
||||||
|
// Get all jobs
|
||||||
|
scraperRouter.get('/jobs', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const limit = parseInt(req.query.limit as string) || 50;
|
||||||
|
|
||||||
|
const jobs = await ScraperJob.find()
|
||||||
|
.sort({ createdAt: -1 })
|
||||||
|
.limit(limit)
|
||||||
|
.lean();
|
||||||
|
|
||||||
|
// Populate target names
|
||||||
|
const targetIds = [...new Set(jobs.filter(j => j.targetId).map(j => j.targetId!.toString()))];
|
||||||
|
const targets = await Target.find({ _id: { $in: targetIds } }).select('name').lean();
|
||||||
|
const targetMap = new Map(targets.map(t => [t._id.toString(), t.name]));
|
||||||
|
|
||||||
|
const formattedJobs = jobs.map(job => ({
|
||||||
|
id: job._id,
|
||||||
|
target_id: job.targetId,
|
||||||
|
profile_id: job.profileId,
|
||||||
|
platform: job.platform,
|
||||||
|
status: job.status,
|
||||||
|
progress: job.progress,
|
||||||
|
result: job.result ? JSON.stringify(job.result) : null,
|
||||||
|
error: job.error,
|
||||||
|
target_name: job.targetId ? targetMap.get(job.targetId.toString()) : null,
|
||||||
|
started_at: job.startedAt,
|
||||||
|
completed_at: job.completedAt,
|
||||||
|
created_at: job.createdAt,
|
||||||
|
}));
|
||||||
|
|
||||||
|
res.json(formattedJobs);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching jobs:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch jobs' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get job by ID with logs
|
||||||
|
scraperRouter.get('/jobs/:id', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const job = await ScraperJob.findById(id).lean();
|
||||||
|
|
||||||
|
if (!job) {
|
||||||
|
res.status(404).json({ error: 'Job not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get target name
|
||||||
|
let targetName = null;
|
||||||
|
if (job.targetId) {
|
||||||
|
const target = await Target.findById(job.targetId).select('name').lean();
|
||||||
|
targetName = target?.name;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
id: job._id,
|
||||||
|
target_id: job.targetId,
|
||||||
|
profile_id: job.profileId,
|
||||||
|
platform: job.platform,
|
||||||
|
status: job.status,
|
||||||
|
progress: job.progress,
|
||||||
|
result: job.result,
|
||||||
|
error: job.error,
|
||||||
|
target_name: targetName,
|
||||||
|
started_at: job.startedAt,
|
||||||
|
completed_at: job.completedAt,
|
||||||
|
created_at: job.createdAt,
|
||||||
|
logs: job.logs,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching job:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch job' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start a new scrape job
|
||||||
|
scraperRouter.post('/start', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { target_id, profile_id, platform, profile_url } = req.body;
|
||||||
|
|
||||||
|
if (!platform) {
|
||||||
|
res.status(400).json({ error: 'Platform is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create job record
|
||||||
|
const job = new ScraperJob({
|
||||||
|
targetId: target_id ? new mongoose.Types.ObjectId(target_id) : undefined,
|
||||||
|
profileId: profile_id ? new mongoose.Types.ObjectId(profile_id) : undefined,
|
||||||
|
platform,
|
||||||
|
status: 'pending',
|
||||||
|
progress: 0,
|
||||||
|
logs: [],
|
||||||
|
});
|
||||||
|
await job.save();
|
||||||
|
|
||||||
|
// Get scraper manager and start job
|
||||||
|
const scraperManager = req.app.get('scraperManager') as ScraperManager;
|
||||||
|
|
||||||
|
scraperManager.startJob({
|
||||||
|
jobId: job._id.toString(),
|
||||||
|
platform,
|
||||||
|
profileUrl: profile_url,
|
||||||
|
targetId: target_id,
|
||||||
|
profileId: profile_id,
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info(`Started scraper job: ${job._id} for ${platform}`);
|
||||||
|
res.status(201).json({
|
||||||
|
id: job._id,
|
||||||
|
platform: job.platform,
|
||||||
|
status: job.status,
|
||||||
|
progress: job.progress,
|
||||||
|
created_at: job.createdAt,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error starting job:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to start job' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Cancel a job
|
||||||
|
scraperRouter.post('/jobs/:id/cancel', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
|
||||||
|
const scraperManager = req.app.get('scraperManager') as ScraperManager;
|
||||||
|
await scraperManager.cancelJob(id);
|
||||||
|
|
||||||
|
const job = await ScraperJob.findById(id).lean();
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
id: job?._id,
|
||||||
|
status: job?.status,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error cancelling job:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to cancel job' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get active jobs summary
|
||||||
|
scraperRouter.get('/status', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const [pending, running, completed, failed] = await Promise.all([
|
||||||
|
ScraperJob.countDocuments({ status: 'pending' }),
|
||||||
|
ScraperJob.countDocuments({ status: 'running' }),
|
||||||
|
ScraperJob.countDocuments({ status: 'completed' }),
|
||||||
|
ScraperJob.countDocuments({ status: 'failed' }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const activeJobs = await ScraperJob.find({ status: { $in: ['pending', 'running'] } })
|
||||||
|
.sort({ createdAt: -1 })
|
||||||
|
.select('_id platform status progress startedAt')
|
||||||
|
.lean();
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
counts: { pending, running, completed, failed },
|
||||||
|
activeJobs: activeJobs.map(j => ({
|
||||||
|
id: j._id,
|
||||||
|
platform: j.platform,
|
||||||
|
status: j.status,
|
||||||
|
progress: j.progress,
|
||||||
|
started_at: j.startedAt,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching scraper status:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch status' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get logs for a job
|
||||||
|
scraperRouter.get('/jobs/:id/logs', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const job = await ScraperJob.findById(id).select('logs').lean();
|
||||||
|
|
||||||
|
if (!job) {
|
||||||
|
res.status(404).json({ error: 'Job not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json(job.logs || []);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching logs:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch logs' });
|
||||||
|
}
|
||||||
|
});
|
||||||
225
backend/src/routes/sessions.ts
Normal file
225
backend/src/routes/sessions.ts
Normal file
@@ -0,0 +1,225 @@
|
|||||||
|
import { Router, Request, Response } from 'express';
|
||||||
|
import { Session } from '../models/Session.js';
|
||||||
|
import { encrypt, decrypt } from '../utils/encryption.js';
|
||||||
|
import { logger } from '../utils/logger.js';
|
||||||
|
|
||||||
|
export const sessionsRouter = Router();
|
||||||
|
|
||||||
|
// Get all sessions (without sensitive data)
|
||||||
|
sessionsRouter.get('/', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const sessions = await Session.find()
|
||||||
|
.select('-cookiesEncrypted -localStorageEncrypted')
|
||||||
|
.sort({ updatedAt: -1 })
|
||||||
|
.lean();
|
||||||
|
|
||||||
|
const formattedSessions = sessions.map(s => ({
|
||||||
|
id: s._id,
|
||||||
|
platform: s.platform,
|
||||||
|
session_name: s.sessionName,
|
||||||
|
user_agent: s.userAgent,
|
||||||
|
proxy: s.proxy,
|
||||||
|
status: s.status,
|
||||||
|
last_validated: s.lastValidated,
|
||||||
|
created_at: s.createdAt,
|
||||||
|
updated_at: s.updatedAt,
|
||||||
|
}));
|
||||||
|
|
||||||
|
res.json(formattedSessions);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching sessions:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch sessions' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get session by platform
|
||||||
|
sessionsRouter.get('/platform/:platform', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { platform } = req.params;
|
||||||
|
const session = await Session.findOne({ platform })
|
||||||
|
.select('-cookiesEncrypted -localStorageEncrypted')
|
||||||
|
.lean();
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
res.status(404).json({ error: 'Session not found for platform' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
id: session._id,
|
||||||
|
platform: session.platform,
|
||||||
|
session_name: session.sessionName,
|
||||||
|
user_agent: session.userAgent,
|
||||||
|
proxy: session.proxy,
|
||||||
|
status: session.status,
|
||||||
|
last_validated: session.lastValidated,
|
||||||
|
created_at: session.createdAt,
|
||||||
|
updated_at: session.updatedAt,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching session:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch session' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create or update session
|
||||||
|
sessionsRouter.post('/', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { platform, session_name, cookies, local_storage, user_agent, proxy } = req.body;
|
||||||
|
|
||||||
|
if (!platform || !session_name || !cookies) {
|
||||||
|
res.status(400).json({ error: 'Platform, session_name, and cookies are required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encrypt sensitive data
|
||||||
|
const cookiesEncrypted = encrypt(JSON.stringify(cookies));
|
||||||
|
const localStorageEncrypted = local_storage ? encrypt(JSON.stringify(local_storage)) : undefined;
|
||||||
|
|
||||||
|
// Upsert session
|
||||||
|
const session = await Session.findOneAndUpdate(
|
||||||
|
{ platform },
|
||||||
|
{
|
||||||
|
platform,
|
||||||
|
sessionName: session_name,
|
||||||
|
cookiesEncrypted,
|
||||||
|
localStorageEncrypted,
|
||||||
|
userAgent: user_agent,
|
||||||
|
proxy,
|
||||||
|
status: 'active',
|
||||||
|
},
|
||||||
|
{ upsert: true, new: true }
|
||||||
|
).lean();
|
||||||
|
|
||||||
|
logger.info(`Saved session for ${platform}`);
|
||||||
|
|
||||||
|
res.status(201).json({
|
||||||
|
id: session._id,
|
||||||
|
platform: session.platform,
|
||||||
|
session_name: session.sessionName,
|
||||||
|
user_agent: session.userAgent,
|
||||||
|
proxy: session.proxy,
|
||||||
|
status: session.status,
|
||||||
|
created_at: session.createdAt,
|
||||||
|
updated_at: session.updatedAt,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error saving session:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to save session' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get decrypted session data (for internal scraper use)
|
||||||
|
sessionsRouter.get('/decrypt/:platform', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { platform } = req.params;
|
||||||
|
const session = await Session.findOne({ platform }).lean();
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
res.status(404).json({ error: 'Session not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = JSON.parse(decrypt(session.cookiesEncrypted));
|
||||||
|
const localStorage = session.localStorageEncrypted
|
||||||
|
? JSON.parse(decrypt(session.localStorageEncrypted))
|
||||||
|
: null;
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
id: session._id,
|
||||||
|
platform: session.platform,
|
||||||
|
session_name: session.sessionName,
|
||||||
|
cookies,
|
||||||
|
localStorage,
|
||||||
|
user_agent: session.userAgent,
|
||||||
|
proxy: session.proxy,
|
||||||
|
status: session.status,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error decrypting session:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to decrypt session' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Validate session
|
||||||
|
sessionsRouter.post('/validate/:platform', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { platform } = req.params;
|
||||||
|
|
||||||
|
const session = await Session.findOneAndUpdate(
|
||||||
|
{ platform },
|
||||||
|
{ lastValidated: new Date() },
|
||||||
|
{ new: true }
|
||||||
|
).select('-cookiesEncrypted -localStorageEncrypted').lean();
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
res.status(404).json({ error: 'Session not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
id: session._id,
|
||||||
|
platform: session.platform,
|
||||||
|
session_name: session.sessionName,
|
||||||
|
status: session.status,
|
||||||
|
last_validated: session.lastValidated,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error validating session:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to validate session' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Update session status
|
||||||
|
sessionsRouter.patch('/:id/status', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const { status } = req.body;
|
||||||
|
|
||||||
|
if (!status || !['active', 'expired', 'invalid'].includes(status)) {
|
||||||
|
res.status(400).json({ error: 'Valid status required (active, expired, invalid)' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const session = await Session.findByIdAndUpdate(
|
||||||
|
id,
|
||||||
|
{ status },
|
||||||
|
{ new: true }
|
||||||
|
).select('-cookiesEncrypted -localStorageEncrypted').lean();
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
res.status(404).json({ error: 'Session not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
id: session._id,
|
||||||
|
platform: session.platform,
|
||||||
|
session_name: session.sessionName,
|
||||||
|
status: session.status,
|
||||||
|
updated_at: session.updatedAt,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error updating session status:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to update session status' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Delete session
|
||||||
|
sessionsRouter.delete('/:id', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const result = await Session.findByIdAndDelete(id);
|
||||||
|
|
||||||
|
if (!result) {
|
||||||
|
res.status(404).json({ error: 'Session not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Deleted session: ${id}`);
|
||||||
|
res.json({ success: true });
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error deleting session:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to delete session' });
|
||||||
|
}
|
||||||
|
});
|
||||||
199
backend/src/routes/targets.ts
Normal file
199
backend/src/routes/targets.ts
Normal file
@@ -0,0 +1,199 @@
|
|||||||
|
import { Router, Request, Response } from 'express';
|
||||||
|
import { Target } from '../models/Target.js';
|
||||||
|
import { logger } from '../utils/logger.js';
|
||||||
|
|
||||||
|
export const targetsRouter = Router();
|
||||||
|
|
||||||
|
// Get all targets
|
||||||
|
targetsRouter.get('/', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const targets = await Target.find()
|
||||||
|
.sort({ updatedAt: -1 })
|
||||||
|
.lean();
|
||||||
|
|
||||||
|
// Add profile count
|
||||||
|
const targetsWithCount = targets.map(t => ({
|
||||||
|
...t,
|
||||||
|
id: t._id,
|
||||||
|
profile_count: t.profiles?.length || 0,
|
||||||
|
}));
|
||||||
|
|
||||||
|
res.json(targetsWithCount);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching targets:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch targets' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get single target with profiles
|
||||||
|
targetsRouter.get('/:id', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const target = await Target.findById(id).lean();
|
||||||
|
|
||||||
|
if (!target) {
|
||||||
|
res.status(404).json({ error: 'Target not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
...target,
|
||||||
|
id: target._id,
|
||||||
|
profiles: target.profiles?.map(p => ({
|
||||||
|
...p,
|
||||||
|
id: p._id,
|
||||||
|
target_id: target._id,
|
||||||
|
profile_url: p.profileUrl,
|
||||||
|
profile_data: p.profileData ? JSON.stringify(p.profileData) : null,
|
||||||
|
last_scraped: p.lastScraped,
|
||||||
|
created_at: p.createdAt,
|
||||||
|
})) || [],
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error fetching target:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to fetch target' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create target
|
||||||
|
targetsRouter.post('/', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { name, notes } = req.body;
|
||||||
|
|
||||||
|
if (!name) {
|
||||||
|
res.status(400).json({ error: 'Name is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const target = new Target({ name, notes, profiles: [] });
|
||||||
|
await target.save();
|
||||||
|
|
||||||
|
logger.info(`Created target: ${name} (${target._id})`);
|
||||||
|
res.status(201).json({
|
||||||
|
...target.toObject(),
|
||||||
|
id: target._id,
|
||||||
|
profile_count: 0,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error creating target:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to create target' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Update target
|
||||||
|
targetsRouter.put('/:id', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const { name, notes } = req.body;
|
||||||
|
|
||||||
|
const target = await Target.findByIdAndUpdate(
|
||||||
|
id,
|
||||||
|
{ name, notes },
|
||||||
|
{ new: true }
|
||||||
|
).lean();
|
||||||
|
|
||||||
|
if (!target) {
|
||||||
|
res.status(404).json({ error: 'Target not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({ ...target, id: target._id });
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error updating target:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to update target' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Delete target
|
||||||
|
targetsRouter.delete('/:id', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const result = await Target.findByIdAndDelete(id);
|
||||||
|
|
||||||
|
if (!result) {
|
||||||
|
res.status(404).json({ error: 'Target not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Deleted target: ${id}`);
|
||||||
|
res.json({ success: true });
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error deleting target:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to delete target' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Add profile to target
|
||||||
|
targetsRouter.post('/:id/profiles', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id } = req.params;
|
||||||
|
const { platform, username, profile_url } = req.body;
|
||||||
|
|
||||||
|
if (!platform) {
|
||||||
|
res.status(400).json({ error: 'Platform is required' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const target = await Target.findById(id);
|
||||||
|
if (!target) {
|
||||||
|
res.status(404).json({ error: 'Target not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const newProfile = {
|
||||||
|
platform,
|
||||||
|
username: username || undefined,
|
||||||
|
profileUrl: profile_url || undefined,
|
||||||
|
createdAt: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
target.profiles.push(newProfile as any);
|
||||||
|
await target.save();
|
||||||
|
|
||||||
|
const addedProfile = target.profiles[target.profiles.length - 1];
|
||||||
|
|
||||||
|
logger.info(`Added ${platform} profile to target ${id}`);
|
||||||
|
res.status(201).json({
|
||||||
|
id: addedProfile._id,
|
||||||
|
target_id: id,
|
||||||
|
platform: addedProfile.platform,
|
||||||
|
username: addedProfile.username,
|
||||||
|
profile_url: addedProfile.profileUrl,
|
||||||
|
created_at: addedProfile.createdAt,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error adding profile:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to add profile' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Delete profile
|
||||||
|
targetsRouter.delete('/:id/profiles/:profileId', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const { id, profileId } = req.params;
|
||||||
|
|
||||||
|
const target = await Target.findById(id);
|
||||||
|
if (!target) {
|
||||||
|
res.status(404).json({ error: 'Target not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const profileIndex = target.profiles.findIndex(
|
||||||
|
p => p._id.toString() === profileId
|
||||||
|
);
|
||||||
|
|
||||||
|
if (profileIndex === -1) {
|
||||||
|
res.status(404).json({ error: 'Profile not found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
target.profiles.splice(profileIndex, 1);
|
||||||
|
await target.save();
|
||||||
|
|
||||||
|
logger.info(`Deleted profile ${profileId} from target ${id}`);
|
||||||
|
res.json({ success: true });
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error deleting profile:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to delete profile' });
|
||||||
|
}
|
||||||
|
});
|
||||||
509
backend/src/scraper/manager.ts
Normal file
509
backend/src/scraper/manager.ts
Normal file
@@ -0,0 +1,509 @@
|
|||||||
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||||
|
import { Server } from 'socket.io';
|
||||||
|
import { chromium, Browser, BrowserContext, Page } from 'playwright';
|
||||||
|
import mongoose from 'mongoose';
|
||||||
|
import { ScraperJob } from '../models/ScraperJob.js';
|
||||||
|
import { Session } from '../models/Session.js';
|
||||||
|
import { Target } from '../models/Target.js';
|
||||||
|
import { decrypt } from '../utils/encryption.js';
|
||||||
|
import { logger } from '../utils/logger.js';
|
||||||
|
|
||||||
|
declare const document: any;
|
||||||
|
declare const window: any;
|
||||||
|
|
||||||
|
export interface ScraperJobConfig {
|
||||||
|
jobId: string;
|
||||||
|
platform: string;
|
||||||
|
profileUrl?: string;
|
||||||
|
targetId?: string;
|
||||||
|
profileId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ActiveJob {
|
||||||
|
config: ScraperJobConfig;
|
||||||
|
browser?: Browser;
|
||||||
|
context?: BrowserContext;
|
||||||
|
page?: Page;
|
||||||
|
abortController: AbortController;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ScraperManager {
|
||||||
|
private io: Server;
|
||||||
|
private activeJobs: Map<string, ActiveJob> = new Map();
|
||||||
|
private browser: Browser | null = null;
|
||||||
|
|
||||||
|
constructor(io: Server) {
|
||||||
|
this.io = io;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async log(jobId: string, level: string, message: string): Promise<void> {
|
||||||
|
await ScraperJob.findByIdAndUpdate(jobId, {
|
||||||
|
$push: {
|
||||||
|
logs: {
|
||||||
|
level,
|
||||||
|
message,
|
||||||
|
timestamp: new Date(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Emit to socket
|
||||||
|
this.io.to(`scraper:${jobId}`).emit('scraper:log', {
|
||||||
|
jobId,
|
||||||
|
level,
|
||||||
|
message,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
|
||||||
|
logger[level as 'info' | 'warn' | 'error'](`[Job ${jobId}] ${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async updateJobStatus(
|
||||||
|
jobId: string,
|
||||||
|
status: string,
|
||||||
|
progress?: number,
|
||||||
|
result?: any,
|
||||||
|
error?: string
|
||||||
|
): Promise<void> {
|
||||||
|
const update: any = { status };
|
||||||
|
|
||||||
|
if (progress !== undefined) update.progress = progress;
|
||||||
|
if (result !== undefined) update.result = result;
|
||||||
|
if (error !== undefined) update.error = error;
|
||||||
|
if (status === 'running') update.startedAt = new Date();
|
||||||
|
if (status === 'completed' || status === 'failed' || status === 'cancelled') {
|
||||||
|
update.completedAt = new Date();
|
||||||
|
}
|
||||||
|
|
||||||
|
await ScraperJob.findByIdAndUpdate(jobId, update);
|
||||||
|
|
||||||
|
// Emit status update
|
||||||
|
this.io.to(`scraper:${jobId}`).emit('scraper:status', {
|
||||||
|
jobId,
|
||||||
|
status,
|
||||||
|
progress,
|
||||||
|
result: result ? result : undefined,
|
||||||
|
error,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Also emit to general channel
|
||||||
|
this.io.emit('scraper:jobUpdate', { jobId, status, progress });
|
||||||
|
}
|
||||||
|
|
||||||
|
async startJob(config: ScraperJobConfig): Promise<void> {
|
||||||
|
const abortController = new AbortController();
|
||||||
|
|
||||||
|
const activeJob: ActiveJob = {
|
||||||
|
config,
|
||||||
|
abortController,
|
||||||
|
};
|
||||||
|
|
||||||
|
this.activeJobs.set(config.jobId, activeJob);
|
||||||
|
|
||||||
|
// Run asynchronously
|
||||||
|
this.runJob(config, abortController.signal).catch((error) => {
|
||||||
|
logger.error(`Job ${config.jobId} failed:`, error);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runJob(config: ScraperJobConfig, signal: AbortSignal): Promise<void> {
|
||||||
|
const { jobId, platform, profileUrl } = config;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.updateJobStatus(jobId, 'running', 0);
|
||||||
|
await this.log(jobId, 'info', `Starting scrape for platform: ${platform}`);
|
||||||
|
|
||||||
|
// Load session from vault
|
||||||
|
const session = await this.loadSession(platform);
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
throw new Error(`No session found for platform: ${platform}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.log(jobId, 'info', `Loaded session: ${session.sessionName}`);
|
||||||
|
await this.updateJobStatus(jobId, 'running', 10);
|
||||||
|
|
||||||
|
// Initialize browser
|
||||||
|
await this.log(jobId, 'info', 'Initializing browser...');
|
||||||
|
const browser = await this.getBrowser();
|
||||||
|
|
||||||
|
if (signal.aborted) {
|
||||||
|
throw new Error('Job cancelled');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create context with session
|
||||||
|
const context = await browser.newContext({
|
||||||
|
userAgent: session.userAgent || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||||
|
viewport: { width: 1920, height: 1080 },
|
||||||
|
locale: 'en-US',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Add cookies
|
||||||
|
if (session.cookies && Array.isArray(session.cookies)) {
|
||||||
|
await context.addCookies(session.cookies);
|
||||||
|
await this.log(jobId, 'info', `Loaded ${session.cookies.length} cookies`);
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'running', 20);
|
||||||
|
|
||||||
|
const page = await context.newPage();
|
||||||
|
|
||||||
|
// Store references
|
||||||
|
const activeJob = this.activeJobs.get(jobId);
|
||||||
|
if (activeJob) {
|
||||||
|
activeJob.context = context;
|
||||||
|
activeJob.page = page;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (signal.aborted) {
|
||||||
|
await context.close();
|
||||||
|
throw new Error('Job cancelled');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run platform-specific scraper
|
||||||
|
let result: any;
|
||||||
|
|
||||||
|
switch (platform.toLowerCase()) {
|
||||||
|
case 'x':
|
||||||
|
case 'twitter':
|
||||||
|
result = await this.scrapeTwitter(jobId, page, profileUrl, signal);
|
||||||
|
break;
|
||||||
|
case 'instagram':
|
||||||
|
result = await this.scrapeInstagram(jobId, page, profileUrl, signal);
|
||||||
|
break;
|
||||||
|
case 'linkedin':
|
||||||
|
result = await this.scrapeLinkedIn(jobId, page, profileUrl, signal);
|
||||||
|
break;
|
||||||
|
case 'facebook':
|
||||||
|
result = await this.scrapeFacebook(jobId, page, profileUrl, signal);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
result = await this.scrapeGeneric(jobId, page, profileUrl || '', signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close context
|
||||||
|
await context.close();
|
||||||
|
|
||||||
|
// Save result to profile if profile_id provided
|
||||||
|
if (config.profileId && config.targetId && result) {
|
||||||
|
await Target.updateOne(
|
||||||
|
{ _id: new mongoose.Types.ObjectId(config.targetId), 'profiles._id': new mongoose.Types.ObjectId(config.profileId) },
|
||||||
|
{
|
||||||
|
$set: {
|
||||||
|
'profiles.$.profileData': result,
|
||||||
|
'profiles.$.lastScraped': new Date()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'completed', 100, result);
|
||||||
|
await this.log(jobId, 'info', 'Scrape completed successfully');
|
||||||
|
|
||||||
|
} catch (error: any) {
|
||||||
|
const errorMessage = error.message || 'Unknown error';
|
||||||
|
await this.log(jobId, 'error', `Scrape failed: ${errorMessage}`);
|
||||||
|
await this.updateJobStatus(jobId, 'failed', undefined, undefined, errorMessage);
|
||||||
|
} finally {
|
||||||
|
this.activeJobs.delete(jobId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async loadSession(platform: string): Promise<any> {
|
||||||
|
const session = await Session.findOne({ platform, status: 'active' }).lean();
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
...session,
|
||||||
|
cookies: JSON.parse(decrypt(session.cookiesEncrypted)),
|
||||||
|
localStorage: session.localStorageEncrypted
|
||||||
|
? JSON.parse(decrypt(session.localStorageEncrypted))
|
||||||
|
: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getBrowser(): Promise<Browser> {
|
||||||
|
if (!this.browser || !this.browser.isConnected()) {
|
||||||
|
this.browser = await chromium.launch({
|
||||||
|
headless: true,
|
||||||
|
args: [
|
||||||
|
'--disable-blink-features=AutomationControlled',
|
||||||
|
'--no-sandbox',
|
||||||
|
'--disable-setuid-sandbox',
|
||||||
|
'--disable-web-security',
|
||||||
|
'--disable-features=IsolateOrigins,site-per-process',
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return this.browser;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Platform-specific scrapers
|
||||||
|
private async scrapeTwitter(
|
||||||
|
jobId: string,
|
||||||
|
page: Page,
|
||||||
|
profileUrl: string | undefined,
|
||||||
|
signal: AbortSignal
|
||||||
|
): Promise<any> {
|
||||||
|
const url = profileUrl || 'https://x.com/home';
|
||||||
|
|
||||||
|
await this.log(jobId, 'info', `Navigating to: ${url}`);
|
||||||
|
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
|
||||||
|
await this.updateJobStatus(jobId, 'running', 40);
|
||||||
|
|
||||||
|
if (signal.aborted) throw new Error('Job cancelled');
|
||||||
|
|
||||||
|
// Wait for content to load
|
||||||
|
await page.waitForTimeout(2000);
|
||||||
|
|
||||||
|
await this.log(jobId, 'info', 'Extracting profile data...');
|
||||||
|
await this.updateJobStatus(jobId, 'running', 60);
|
||||||
|
|
||||||
|
// Extract profile data
|
||||||
|
const profileData = await page.evaluate(() => {
|
||||||
|
const doc = document as any;
|
||||||
|
const win = window as any;
|
||||||
|
const data: any = {
|
||||||
|
url: win.location.href,
|
||||||
|
scraped_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Try to extract profile info
|
||||||
|
const nameElement = doc.querySelector('[data-testid="UserName"]');
|
||||||
|
if (nameElement) {
|
||||||
|
data.display_name = nameElement.querySelector('span')?.textContent;
|
||||||
|
data.username = nameElement.querySelectorAll('span')[1]?.textContent;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bioElement = doc.querySelector('[data-testid="UserDescription"]');
|
||||||
|
if (bioElement) {
|
||||||
|
data.bio = bioElement.textContent;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract stats
|
||||||
|
const statsElements = doc.querySelectorAll('[href*="/following"], [href*="/followers"]');
|
||||||
|
statsElements.forEach((el: any) => {
|
||||||
|
const href = el.getAttribute('href');
|
||||||
|
const text = el.textContent;
|
||||||
|
if (href?.includes('following')) {
|
||||||
|
data.following = text;
|
||||||
|
} else if (href?.includes('followers')) {
|
||||||
|
data.followers = text;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get recent tweets
|
||||||
|
const tweets: any[] = [];
|
||||||
|
doc.querySelectorAll('[data-testid="tweet"]').forEach((tweet: any, i: number) => {
|
||||||
|
if (i < 10) { // Limit to 10 tweets
|
||||||
|
tweets.push({
|
||||||
|
text: tweet.querySelector('[data-testid="tweetText"]')?.textContent,
|
||||||
|
timestamp: tweet.querySelector('time')?.getAttribute('datetime'),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
data.recent_tweets = tweets;
|
||||||
|
|
||||||
|
return data;
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'running', 80);
|
||||||
|
await this.log(jobId, 'info', `Extracted profile: ${profileData.username || 'unknown'}`);
|
||||||
|
|
||||||
|
return profileData;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async scrapeInstagram(
|
||||||
|
jobId: string,
|
||||||
|
page: Page,
|
||||||
|
profileUrl: string | undefined,
|
||||||
|
signal: AbortSignal
|
||||||
|
): Promise<any> {
|
||||||
|
const url = profileUrl || 'https://instagram.com';
|
||||||
|
|
||||||
|
await this.log(jobId, 'info', `Navigating to: ${url}`);
|
||||||
|
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
|
||||||
|
await this.updateJobStatus(jobId, 'running', 40);
|
||||||
|
|
||||||
|
if (signal.aborted) throw new Error('Job cancelled');
|
||||||
|
|
||||||
|
await page.waitForTimeout(2000);
|
||||||
|
|
||||||
|
await this.log(jobId, 'info', 'Extracting profile data...');
|
||||||
|
await this.updateJobStatus(jobId, 'running', 60);
|
||||||
|
|
||||||
|
const profileData = await page.evaluate(() => {
|
||||||
|
const doc = document as any;
|
||||||
|
const win = window as any;
|
||||||
|
const data: any = {
|
||||||
|
url: win.location.href,
|
||||||
|
scraped_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Extract from meta tags and visible elements
|
||||||
|
const ogTitle = doc.querySelector('meta[property="og:title"]');
|
||||||
|
if (ogTitle) {
|
||||||
|
data.title = ogTitle.getAttribute('content');
|
||||||
|
}
|
||||||
|
|
||||||
|
const ogDescription = document.querySelector('meta[property="og:description"]');
|
||||||
|
if (ogDescription) {
|
||||||
|
data.description = ogDescription.getAttribute('content');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to parse stats from description
|
||||||
|
const statsMatch = data.description?.match(/(\d+(?:,\d+)*(?:\.\d+)?[KMB]?)\s+Followers/i);
|
||||||
|
if (statsMatch) {
|
||||||
|
data.followers = statsMatch[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get profile picture
|
||||||
|
const profilePic = doc.querySelector('img[alt*="profile picture"]');
|
||||||
|
if (profilePic) {
|
||||||
|
data.profile_picture = profilePic.getAttribute('src');
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'running', 80);
|
||||||
|
|
||||||
|
return profileData;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async scrapeLinkedIn(
|
||||||
|
jobId: string,
|
||||||
|
page: Page,
|
||||||
|
profileUrl: string | undefined,
|
||||||
|
signal: AbortSignal
|
||||||
|
): Promise<any> {
|
||||||
|
const url = profileUrl || 'https://linkedin.com';
|
||||||
|
|
||||||
|
await this.log(jobId, 'info', `Navigating to: ${url}`);
|
||||||
|
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
|
||||||
|
await this.updateJobStatus(jobId, 'running', 40);
|
||||||
|
|
||||||
|
if (signal.aborted) throw new Error('Job cancelled');
|
||||||
|
|
||||||
|
await page.waitForTimeout(2000);
|
||||||
|
await this.updateJobStatus(jobId, 'running', 60);
|
||||||
|
|
||||||
|
const profileData = await page.evaluate(() => {
|
||||||
|
const doc = document as any;
|
||||||
|
const win = window as any;
|
||||||
|
const data: any = {
|
||||||
|
url: win.location.href,
|
||||||
|
scraped_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Extract profile info
|
||||||
|
const nameElement = doc.querySelector('h1');
|
||||||
|
if (nameElement) {
|
||||||
|
data.name = nameElement.textContent?.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
const headlineElement = document.querySelector('.text-body-medium');
|
||||||
|
if (headlineElement) {
|
||||||
|
data.headline = headlineElement.textContent?.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'running', 80);
|
||||||
|
|
||||||
|
return profileData;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async scrapeFacebook(
|
||||||
|
jobId: string,
|
||||||
|
page: Page,
|
||||||
|
profileUrl: string | undefined,
|
||||||
|
signal: AbortSignal
|
||||||
|
): Promise<any> {
|
||||||
|
const url = profileUrl || 'https://facebook.com';
|
||||||
|
|
||||||
|
await this.log(jobId, 'info', `Navigating to: ${url}`);
|
||||||
|
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
|
||||||
|
await this.updateJobStatus(jobId, 'running', 40);
|
||||||
|
|
||||||
|
if (signal.aborted) throw new Error('Job cancelled');
|
||||||
|
|
||||||
|
await page.waitForTimeout(2000);
|
||||||
|
await this.updateJobStatus(jobId, 'running', 60);
|
||||||
|
|
||||||
|
const profileData = await page.evaluate(() => {
|
||||||
|
const doc = document as any;
|
||||||
|
const win = window as any;
|
||||||
|
return {
|
||||||
|
url: win.location.href,
|
||||||
|
scraped_at: new Date().toISOString(),
|
||||||
|
title: doc.title,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'running', 80);
|
||||||
|
|
||||||
|
return profileData;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async scrapeGeneric(
|
||||||
|
jobId: string,
|
||||||
|
page: Page,
|
||||||
|
url: string,
|
||||||
|
signal: AbortSignal
|
||||||
|
): Promise<any> {
|
||||||
|
await this.log(jobId, 'info', `Navigating to: ${url}`);
|
||||||
|
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
|
||||||
|
await this.updateJobStatus(jobId, 'running', 50);
|
||||||
|
|
||||||
|
if (signal.aborted) throw new Error('Job cancelled');
|
||||||
|
|
||||||
|
const data = await page.evaluate(() => {
|
||||||
|
const doc = document as any;
|
||||||
|
const win = window as any;
|
||||||
|
return {
|
||||||
|
url: win.location.href,
|
||||||
|
title: doc.title,
|
||||||
|
scraped_at: new Date().toISOString(),
|
||||||
|
text_content: doc.body.innerText.substring(0, 5000),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'running', 80);
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
async cancelJob(jobId: string): Promise<void> {
|
||||||
|
const activeJob = this.activeJobs.get(jobId);
|
||||||
|
|
||||||
|
if (activeJob) {
|
||||||
|
activeJob.abortController.abort();
|
||||||
|
|
||||||
|
if (activeJob.context) {
|
||||||
|
await activeJob.context.close().catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.updateJobStatus(jobId, 'cancelled');
|
||||||
|
await this.log(jobId, 'info', 'Job cancelled by user');
|
||||||
|
this.activeJobs.delete(jobId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async shutdown(): Promise<void> {
|
||||||
|
// Cancel all active jobs
|
||||||
|
for (const [jobId] of this.activeJobs) {
|
||||||
|
await this.cancelJob(jobId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close browser
|
||||||
|
if (this.browser) {
|
||||||
|
await this.browser.close();
|
||||||
|
this.browser = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
44
backend/src/utils/encryption.ts
Normal file
44
backend/src/utils/encryption.ts
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
import crypto from 'crypto';
|
||||||
|
|
||||||
|
const ALGORITHM = 'aes-256-gcm';
|
||||||
|
const IV_LENGTH = 16;
|
||||||
|
const AUTH_TAG_LENGTH = 16;
|
||||||
|
|
||||||
|
function getEncryptionKey(): Buffer {
|
||||||
|
const key = process.env.VAULT_ENCRYPTION_KEY;
|
||||||
|
if (!key || key.length !== 64) {
|
||||||
|
throw new Error('VAULT_ENCRYPTION_KEY must be a 64-character hex string');
|
||||||
|
}
|
||||||
|
return Buffer.from(key, 'hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function encrypt(text: string): string {
|
||||||
|
const iv = crypto.randomBytes(IV_LENGTH);
|
||||||
|
const cipher = crypto.createCipheriv(ALGORITHM, getEncryptionKey(), iv);
|
||||||
|
|
||||||
|
let encrypted = cipher.update(text, 'utf8', 'hex');
|
||||||
|
encrypted += cipher.final('hex');
|
||||||
|
|
||||||
|
const authTag = cipher.getAuthTag();
|
||||||
|
|
||||||
|
// Combine IV + AuthTag + Encrypted data
|
||||||
|
return iv.toString('hex') + authTag.toString('hex') + encrypted;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decrypt(encryptedData: string): string {
|
||||||
|
const iv = Buffer.from(encryptedData.slice(0, IV_LENGTH * 2), 'hex');
|
||||||
|
const authTag = Buffer.from(encryptedData.slice(IV_LENGTH * 2, IV_LENGTH * 2 + AUTH_TAG_LENGTH * 2), 'hex');
|
||||||
|
const encrypted = encryptedData.slice(IV_LENGTH * 2 + AUTH_TAG_LENGTH * 2);
|
||||||
|
|
||||||
|
const decipher = crypto.createDecipheriv(ALGORITHM, getEncryptionKey(), iv);
|
||||||
|
decipher.setAuthTag(authTag);
|
||||||
|
|
||||||
|
let decrypted = decipher.update(encrypted, 'hex', 'utf8');
|
||||||
|
decrypted += decipher.final('utf8');
|
||||||
|
|
||||||
|
return decrypted;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hashPassword(password: string): string {
|
||||||
|
return crypto.createHash('sha256').update(password).digest('hex');
|
||||||
|
}
|
||||||
33
backend/src/utils/logger.ts
Normal file
33
backend/src/utils/logger.ts
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
import winston from 'winston';
|
||||||
|
|
||||||
|
const logFormat = winston.format.combine(
|
||||||
|
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
|
||||||
|
winston.format.errors({ stack: true }),
|
||||||
|
winston.format.printf(({ level, message, timestamp, stack }) => {
|
||||||
|
return `${timestamp} [${level.toUpperCase()}]: ${stack || message}`;
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
export const logger = winston.createLogger({
|
||||||
|
level: process.env.NODE_ENV === 'production' ? 'info' : 'debug',
|
||||||
|
format: logFormat,
|
||||||
|
transports: [
|
||||||
|
new winston.transports.Console({
|
||||||
|
format: winston.format.combine(
|
||||||
|
winston.format.colorize(),
|
||||||
|
logFormat
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
new winston.transports.File({
|
||||||
|
filename: 'logs/error.log',
|
||||||
|
level: 'error',
|
||||||
|
maxsize: 5242880, // 5MB
|
||||||
|
maxFiles: 5,
|
||||||
|
}),
|
||||||
|
new winston.transports.File({
|
||||||
|
filename: 'logs/combined.log',
|
||||||
|
maxsize: 5242880,
|
||||||
|
maxFiles: 5,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
20
backend/tsconfig.json
Normal file
20
backend/tsconfig.json
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "NodeNext",
|
||||||
|
"moduleResolution": "NodeNext",
|
||||||
|
"lib": ["ES2022"],
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"strict": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"],
|
||||||
|
"exclude": ["node_modules", "dist"]
|
||||||
|
}
|
||||||
59
docker-compose.yml
Normal file
59
docker-compose.yml
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
osint-platform:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
container_name: osint-platform
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- "3001:3001"
|
||||||
|
environment:
|
||||||
|
- NODE_ENV=production
|
||||||
|
- PORT=3001
|
||||||
|
# REQUIRED: Set these in Coolify or via .env file
|
||||||
|
- MONGODB_URI=${MONGODB_URI}
|
||||||
|
- MASTER_PASSWORD=${MASTER_PASSWORD}
|
||||||
|
- JWT_SECRET=${JWT_SECRET}
|
||||||
|
- VAULT_ENCRYPTION_KEY=${VAULT_ENCRYPTION_KEY}
|
||||||
|
- FRONTEND_URL=${FRONTEND_URL:-http://localhost:3001}
|
||||||
|
- SESSION_EXPIRY=${SESSION_EXPIRY:-24h}
|
||||||
|
volumes:
|
||||||
|
# Logs only (database is MongoDB)
|
||||||
|
- osint-logs:/app/logs
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3001/api/health"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 40s
|
||||||
|
# Security options
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
# Resource limits (adjust based on your needs)
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: 2G
|
||||||
|
reservations:
|
||||||
|
memory: 512M
|
||||||
|
|
||||||
|
# Optional: Local MongoDB for development
|
||||||
|
# Uncomment if you want to run MongoDB locally via Docker
|
||||||
|
# mongodb:
|
||||||
|
# image: mongo:7
|
||||||
|
# container_name: osint-mongodb
|
||||||
|
# restart: unless-stopped
|
||||||
|
# ports:
|
||||||
|
# - "27017:27017"
|
||||||
|
# volumes:
|
||||||
|
# - mongodb-data:/data/db
|
||||||
|
# environment:
|
||||||
|
# - MONGO_INITDB_DATABASE=osint_platform
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
osint-logs:
|
||||||
|
driver: local
|
||||||
|
# mongodb-data:
|
||||||
|
# driver: local
|
||||||
18
frontend/index.html
Normal file
18
frontend/index.html
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="description" content="Private OSINT Automation Platform" />
|
||||||
|
<meta name="theme-color" content="#020617" />
|
||||||
|
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||||
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||||
|
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet" />
|
||||||
|
<title>OSINT Platform</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="root"></div>
|
||||||
|
<script type="module" src="/src/main.tsx"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
2848
frontend/package-lock.json
generated
Normal file
2848
frontend/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
31
frontend/package.json
Normal file
31
frontend/package.json
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"name": "osint-frontend",
|
||||||
|
"private": true,
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vite",
|
||||||
|
"build": "tsc && vite build",
|
||||||
|
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
|
||||||
|
"preview": "vite preview"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"react": "^18.2.0",
|
||||||
|
"react-dom": "^18.2.0",
|
||||||
|
"react-router-dom": "^6.22.3",
|
||||||
|
"socket.io-client": "^4.7.5",
|
||||||
|
"lucide-react": "^0.359.0",
|
||||||
|
"zustand": "^4.5.2",
|
||||||
|
"date-fns": "^3.6.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/react": "^18.2.64",
|
||||||
|
"@types/react-dom": "^18.2.21",
|
||||||
|
"@vitejs/plugin-react": "^4.2.1",
|
||||||
|
"autoprefixer": "^10.4.18",
|
||||||
|
"postcss": "^8.4.35",
|
||||||
|
"tailwindcss": "^3.4.1",
|
||||||
|
"typescript": "^5.4.2",
|
||||||
|
"vite": "^5.1.6"
|
||||||
|
}
|
||||||
|
}
|
||||||
6
frontend/postcss.config.js
Normal file
6
frontend/postcss.config.js
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
export default {
|
||||||
|
plugins: {
|
||||||
|
tailwindcss: {},
|
||||||
|
autoprefixer: {},
|
||||||
|
},
|
||||||
|
}
|
||||||
6
frontend/public/vite.svg
Normal file
6
frontend/public/vite.svg
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" fill="none">
|
||||||
|
<rect width="32" height="32" rx="6" fill="#020617"/>
|
||||||
|
<path d="M16 6L7 11v10l9 5 9-5V11l-9-5z" stroke="#22d3ee" stroke-width="1.5" fill="none"/>
|
||||||
|
<circle cx="16" cy="16" r="4" fill="#22d3ee" opacity="0.3"/>
|
||||||
|
<path d="M16 12v8M12 16h8" stroke="#22d3ee" stroke-width="1.5" stroke-linecap="round"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 387 B |
36
frontend/src/App.tsx
Normal file
36
frontend/src/App.tsx
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom';
|
||||||
|
import { useAuthStore } from './stores/authStore';
|
||||||
|
import { LoginPage } from './pages/LoginPage';
|
||||||
|
import { Dashboard } from './pages/Dashboard';
|
||||||
|
|
||||||
|
function ProtectedRoute({ children }: { children: React.ReactNode }) {
|
||||||
|
const isAuthenticated = useAuthStore((state) => state.isAuthenticated);
|
||||||
|
|
||||||
|
if (!isAuthenticated) {
|
||||||
|
return <Navigate to="/login" replace />;
|
||||||
|
}
|
||||||
|
|
||||||
|
return <>{children}</>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function App() {
|
||||||
|
return (
|
||||||
|
<BrowserRouter future={{ v7_startTransition: true, v7_relativeSplatPath: true }}>
|
||||||
|
<div className="min-h-screen bg-slate-950">
|
||||||
|
<Routes>
|
||||||
|
<Route path="/login" element={<LoginPage />} />
|
||||||
|
<Route
|
||||||
|
path="/*"
|
||||||
|
element={
|
||||||
|
<ProtectedRoute>
|
||||||
|
<Dashboard />
|
||||||
|
</ProtectedRoute>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</Routes>
|
||||||
|
</div>
|
||||||
|
</BrowserRouter>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default App;
|
||||||
161
frontend/src/components/AddProfileModal.tsx
Normal file
161
frontend/src/components/AddProfileModal.tsx
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { useTargetStore } from '../stores/targetStore';
|
||||||
|
import { X, Globe, Twitter, Instagram, Linkedin, Facebook } from 'lucide-react';
|
||||||
|
|
||||||
|
interface AddProfileModalProps {
|
||||||
|
onClose: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const PLATFORMS = [
|
||||||
|
{ id: 'x', name: 'X (Twitter)', icon: Twitter },
|
||||||
|
{ id: 'instagram', name: 'Instagram', icon: Instagram },
|
||||||
|
{ id: 'linkedin', name: 'LinkedIn', icon: Linkedin },
|
||||||
|
{ id: 'facebook', name: 'Facebook', icon: Facebook },
|
||||||
|
{ id: 'other', name: 'Other', icon: Globe },
|
||||||
|
];
|
||||||
|
|
||||||
|
export function AddProfileModal({ onClose }: AddProfileModalProps) {
|
||||||
|
const { selectedTarget, addProfile } = useTargetStore();
|
||||||
|
const [platform, setPlatform] = useState('x');
|
||||||
|
const [username, setUsername] = useState('');
|
||||||
|
const [profileUrl, setProfileUrl] = useState('');
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState('');
|
||||||
|
|
||||||
|
if (!selectedTarget) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSubmit = async (e: React.FormEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
if (!platform) return;
|
||||||
|
|
||||||
|
setLoading(true);
|
||||||
|
setError('');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await addProfile(
|
||||||
|
selectedTarget.id,
|
||||||
|
platform,
|
||||||
|
username.trim() || undefined,
|
||||||
|
profileUrl.trim() || undefined
|
||||||
|
);
|
||||||
|
onClose();
|
||||||
|
} catch (err: any) {
|
||||||
|
setError(err.message || 'Failed to add profile');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="fixed inset-0 z-50 flex items-center justify-center">
|
||||||
|
{/* Backdrop */}
|
||||||
|
<div
|
||||||
|
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||||
|
onClick={onClose}
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Modal */}
|
||||||
|
<div className="relative w-full max-w-md mx-4 bg-slate-900 border border-slate-800 rounded-xl shadow-2xl">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center justify-between p-4 border-b border-slate-800">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Globe className="w-5 h-5 text-accent-cyan" />
|
||||||
|
<div>
|
||||||
|
<h2 className="text-lg font-semibold text-slate-100">Add Profile</h2>
|
||||||
|
<p className="text-xs text-slate-500">for {selectedTarget.name}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={onClose}
|
||||||
|
className="p-1.5 rounded-lg text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
|
||||||
|
>
|
||||||
|
<X className="w-5 h-5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Form */}
|
||||||
|
<form onSubmit={handleSubmit} className="p-4 space-y-4">
|
||||||
|
{error && (
|
||||||
|
<div className="p-3 rounded-lg bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
|
||||||
|
{error}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Platform *
|
||||||
|
</label>
|
||||||
|
<div className="grid grid-cols-5 gap-2">
|
||||||
|
{PLATFORMS.map((p) => {
|
||||||
|
const Icon = p.icon;
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={p.id}
|
||||||
|
type="button"
|
||||||
|
onClick={() => setPlatform(p.id)}
|
||||||
|
className={`flex flex-col items-center gap-1 p-3 rounded-lg border transition-colors ${
|
||||||
|
platform === p.id
|
||||||
|
? 'border-accent-cyan bg-accent-cyan/10 text-accent-cyan'
|
||||||
|
: 'border-slate-700 text-slate-400 hover:border-slate-600 hover:text-slate-300'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<Icon className="w-5 h-5" />
|
||||||
|
<span className="text-[10px]">{p.id === 'x' ? 'X' : p.id.slice(0, 2).toUpperCase()}</span>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="username" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Username
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="username"
|
||||||
|
type="text"
|
||||||
|
value={username}
|
||||||
|
onChange={(e) => setUsername(e.target.value)}
|
||||||
|
className="input-field w-full font-mono"
|
||||||
|
placeholder="@username"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="profileUrl" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Profile URL
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="profileUrl"
|
||||||
|
type="url"
|
||||||
|
value={profileUrl}
|
||||||
|
onChange={(e) => setProfileUrl(e.target.value)}
|
||||||
|
className="input-field w-full font-mono text-sm"
|
||||||
|
placeholder="https://..."
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex gap-3 pt-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onClose}
|
||||||
|
className="btn-secondary flex-1"
|
||||||
|
disabled={loading}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
className="btn-primary flex-1"
|
||||||
|
disabled={!platform || loading}
|
||||||
|
>
|
||||||
|
{loading ? 'Adding...' : 'Add Profile'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
190
frontend/src/components/AddSessionModal.tsx
Normal file
190
frontend/src/components/AddSessionModal.tsx
Normal file
@@ -0,0 +1,190 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { useScraperStore } from '../stores/scraperStore';
|
||||||
|
import { X, Key, AlertCircle, HelpCircle } from 'lucide-react';
|
||||||
|
|
||||||
|
interface AddSessionModalProps {
|
||||||
|
onClose: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const PLATFORMS = [
|
||||||
|
{ id: 'x', name: 'X (Twitter)' },
|
||||||
|
{ id: 'instagram', name: 'Instagram' },
|
||||||
|
{ id: 'linkedin', name: 'LinkedIn' },
|
||||||
|
{ id: 'facebook', name: 'Facebook' },
|
||||||
|
];
|
||||||
|
|
||||||
|
export function AddSessionModal({ onClose }: AddSessionModalProps) {
|
||||||
|
const { addSession } = useScraperStore();
|
||||||
|
const [platform, setPlatform] = useState('x');
|
||||||
|
const [sessionName, setSessionName] = useState('');
|
||||||
|
const [cookiesJson, setCookiesJson] = useState('');
|
||||||
|
const [userAgent, setUserAgent] = useState('');
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState('');
|
||||||
|
|
||||||
|
const handleSubmit = async (e: React.FormEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
|
||||||
|
if (!platform || !sessionName.trim() || !cookiesJson.trim()) {
|
||||||
|
setError('Please fill in all required fields');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse cookies JSON
|
||||||
|
let cookies: any[];
|
||||||
|
try {
|
||||||
|
cookies = JSON.parse(cookiesJson);
|
||||||
|
if (!Array.isArray(cookies)) {
|
||||||
|
throw new Error('Cookies must be an array');
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
setError('Invalid cookies JSON. Please provide a valid array of cookie objects.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setLoading(true);
|
||||||
|
setError('');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await addSession({
|
||||||
|
platform,
|
||||||
|
session_name: sessionName.trim(),
|
||||||
|
cookies,
|
||||||
|
user_agent: userAgent.trim() || undefined,
|
||||||
|
});
|
||||||
|
onClose();
|
||||||
|
} catch (err: any) {
|
||||||
|
setError(err.message || 'Failed to add session');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="fixed inset-0 z-50 flex items-center justify-center">
|
||||||
|
{/* Backdrop */}
|
||||||
|
<div
|
||||||
|
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||||
|
onClick={onClose}
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Modal */}
|
||||||
|
<div className="relative w-full max-w-lg mx-4 bg-slate-900 border border-slate-800 rounded-xl shadow-2xl">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center justify-between p-4 border-b border-slate-800">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Key className="w-5 h-5 text-accent-cyan" />
|
||||||
|
<h2 className="text-lg font-semibold text-slate-100">Add Session</h2>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={onClose}
|
||||||
|
className="p-1.5 rounded-lg text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
|
||||||
|
>
|
||||||
|
<X className="w-5 h-5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Form */}
|
||||||
|
<form onSubmit={handleSubmit} className="p-4 space-y-4">
|
||||||
|
{error && (
|
||||||
|
<div className="flex items-start gap-2 p-3 rounded-lg bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
|
||||||
|
<AlertCircle className="w-4 h-4 mt-0.5 flex-shrink-0" />
|
||||||
|
<span>{error}</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Info box */}
|
||||||
|
<div className="flex items-start gap-2 p-3 rounded-lg bg-accent-cyan/10 border border-accent-cyan/20 text-accent-cyan text-xs">
|
||||||
|
<HelpCircle className="w-4 h-4 mt-0.5 flex-shrink-0" />
|
||||||
|
<div>
|
||||||
|
<p className="font-medium mb-1">How to get cookies:</p>
|
||||||
|
<ol className="list-decimal list-inside text-accent-cyan/80 space-y-0.5">
|
||||||
|
<li>Log into the platform in your browser</li>
|
||||||
|
<li>Open DevTools (F12) → Application → Cookies</li>
|
||||||
|
<li>Export cookies as JSON array</li>
|
||||||
|
<li>Or use a browser extension like "Cookie Editor"</li>
|
||||||
|
</ol>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<label htmlFor="platform" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Platform *
|
||||||
|
</label>
|
||||||
|
<select
|
||||||
|
id="platform"
|
||||||
|
value={platform}
|
||||||
|
onChange={(e) => setPlatform(e.target.value)}
|
||||||
|
className="input-field w-full"
|
||||||
|
>
|
||||||
|
{PLATFORMS.map((p) => (
|
||||||
|
<option key={p.id} value={p.id}>{p.name}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="sessionName" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Session Name *
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="sessionName"
|
||||||
|
type="text"
|
||||||
|
value={sessionName}
|
||||||
|
onChange={(e) => setSessionName(e.target.value)}
|
||||||
|
className="input-field w-full"
|
||||||
|
placeholder="My Account"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="cookies" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Cookies JSON *
|
||||||
|
</label>
|
||||||
|
<textarea
|
||||||
|
id="cookies"
|
||||||
|
value={cookiesJson}
|
||||||
|
onChange={(e) => setCookiesJson(e.target.value)}
|
||||||
|
className="input-field w-full h-32 resize-none font-mono text-xs"
|
||||||
|
placeholder='[{"name": "auth_token", "value": "...", "domain": ".twitter.com"}]'
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="userAgent" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
User Agent <span className="text-slate-600">(optional)</span>
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="userAgent"
|
||||||
|
type="text"
|
||||||
|
value={userAgent}
|
||||||
|
onChange={(e) => setUserAgent(e.target.value)}
|
||||||
|
className="input-field w-full font-mono text-xs"
|
||||||
|
placeholder="Mozilla/5.0 ..."
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex gap-3 pt-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onClose}
|
||||||
|
className="btn-secondary flex-1"
|
||||||
|
disabled={loading}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
className="btn-primary flex-1"
|
||||||
|
disabled={!platform || !sessionName.trim() || !cookiesJson.trim() || loading}
|
||||||
|
>
|
||||||
|
{loading ? 'Saving...' : 'Save Session'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
114
frontend/src/components/AddTargetModal.tsx
Normal file
114
frontend/src/components/AddTargetModal.tsx
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { useTargetStore } from '../stores/targetStore';
|
||||||
|
import { X, UserPlus } from 'lucide-react';
|
||||||
|
|
||||||
|
interface AddTargetModalProps {
|
||||||
|
onClose: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AddTargetModal({ onClose }: AddTargetModalProps) {
|
||||||
|
const { createTarget } = useTargetStore();
|
||||||
|
const [name, setName] = useState('');
|
||||||
|
const [notes, setNotes] = useState('');
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState('');
|
||||||
|
|
||||||
|
const handleSubmit = async (e: React.FormEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
if (!name.trim()) return;
|
||||||
|
|
||||||
|
setLoading(true);
|
||||||
|
setError('');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await createTarget(name.trim(), notes.trim() || undefined);
|
||||||
|
onClose();
|
||||||
|
} catch (err: any) {
|
||||||
|
setError(err.message || 'Failed to create target');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="fixed inset-0 z-50 flex items-center justify-center">
|
||||||
|
{/* Backdrop */}
|
||||||
|
<div
|
||||||
|
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||||
|
onClick={onClose}
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Modal */}
|
||||||
|
<div className="relative w-full max-w-md mx-4 bg-slate-900 border border-slate-800 rounded-xl shadow-2xl">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center justify-between p-4 border-b border-slate-800">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<UserPlus className="w-5 h-5 text-accent-cyan" />
|
||||||
|
<h2 className="text-lg font-semibold text-slate-100">New Target</h2>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={onClose}
|
||||||
|
className="p-1.5 rounded-lg text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
|
||||||
|
>
|
||||||
|
<X className="w-5 h-5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Form */}
|
||||||
|
<form onSubmit={handleSubmit} className="p-4 space-y-4">
|
||||||
|
{error && (
|
||||||
|
<div className="p-3 rounded-lg bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
|
||||||
|
{error}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="name" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Target Name *
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
id="name"
|
||||||
|
type="text"
|
||||||
|
value={name}
|
||||||
|
onChange={(e) => setName(e.target.value)}
|
||||||
|
className="input-field w-full"
|
||||||
|
placeholder="e.g., John Doe"
|
||||||
|
autoFocus
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="notes" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Notes
|
||||||
|
</label>
|
||||||
|
<textarea
|
||||||
|
id="notes"
|
||||||
|
value={notes}
|
||||||
|
onChange={(e) => setNotes(e.target.value)}
|
||||||
|
className="input-field w-full h-24 resize-none"
|
||||||
|
placeholder="Additional information about this target..."
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex gap-3 pt-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onClose}
|
||||||
|
className="btn-secondary flex-1"
|
||||||
|
disabled={loading}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
className="btn-primary flex-1"
|
||||||
|
disabled={!name.trim() || loading}
|
||||||
|
>
|
||||||
|
{loading ? 'Creating...' : 'Create Target'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
86
frontend/src/components/Header.tsx
Normal file
86
frontend/src/components/Header.tsx
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
import { Shield, LogOut, Plus, Key, Activity } from 'lucide-react';
|
||||||
|
import { useAuthStore } from '../stores/authStore';
|
||||||
|
import { useScraperStore } from '../stores/scraperStore';
|
||||||
|
import { useNavigate } from 'react-router-dom';
|
||||||
|
|
||||||
|
interface HeaderProps {
|
||||||
|
onAddTarget: () => void;
|
||||||
|
onAddSession: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function Header({ onAddTarget, onAddSession }: HeaderProps) {
|
||||||
|
const { logout } = useAuthStore();
|
||||||
|
const { activeJobs, sessions } = useScraperStore();
|
||||||
|
const navigate = useNavigate();
|
||||||
|
|
||||||
|
const handleLogout = () => {
|
||||||
|
logout();
|
||||||
|
navigate('/login');
|
||||||
|
};
|
||||||
|
|
||||||
|
const runningJobs = activeJobs.filter(j => j.status === 'running').length;
|
||||||
|
const activeSessions = sessions.filter(s => s.status === 'active').length;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<header className="h-14 flex-shrink-0 bg-slate-900 border-b border-slate-800 px-4 flex items-center justify-between">
|
||||||
|
{/* Logo */}
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
<div className="flex items-center justify-center w-8 h-8 rounded-lg bg-gradient-to-br from-accent-cyan/20 to-accent-violet/20 border border-accent-cyan/30">
|
||||||
|
<Shield className="w-4 h-4 text-accent-cyan" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<h1 className="text-sm font-semibold text-slate-100">OSINT Platform</h1>
|
||||||
|
<p className="text-[10px] text-slate-500 font-mono uppercase tracking-wider">Intelligence Automation</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Status indicators */}
|
||||||
|
<div className="flex items-center gap-4">
|
||||||
|
{/* Running jobs indicator */}
|
||||||
|
<div className="flex items-center gap-2 px-3 py-1.5 rounded-full bg-slate-800/50 border border-slate-700/50">
|
||||||
|
<Activity className={`w-3.5 h-3.5 ${runningJobs > 0 ? 'text-accent-emerald animate-pulse' : 'text-slate-500'}`} />
|
||||||
|
<span className="text-xs font-medium text-slate-400">
|
||||||
|
{runningJobs > 0 ? `${runningJobs} Active` : 'Idle'}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Sessions indicator */}
|
||||||
|
<div className="flex items-center gap-2 px-3 py-1.5 rounded-full bg-slate-800/50 border border-slate-700/50">
|
||||||
|
<Key className={`w-3.5 h-3.5 ${activeSessions > 0 ? 'text-accent-cyan' : 'text-slate-500'}`} />
|
||||||
|
<span className="text-xs font-medium text-slate-400">
|
||||||
|
{activeSessions} Sessions
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Actions */}
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<button
|
||||||
|
onClick={onAddSession}
|
||||||
|
className="btn-secondary flex items-center gap-1.5 text-xs py-1.5"
|
||||||
|
>
|
||||||
|
<Key className="w-3.5 h-3.5" />
|
||||||
|
<span>Add Session</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<button
|
||||||
|
onClick={onAddTarget}
|
||||||
|
className="btn-primary flex items-center gap-1.5 text-xs py-1.5"
|
||||||
|
>
|
||||||
|
<Plus className="w-3.5 h-3.5" />
|
||||||
|
<span>New Target</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<div className="w-px h-6 bg-slate-700 mx-2" />
|
||||||
|
|
||||||
|
<button
|
||||||
|
onClick={handleLogout}
|
||||||
|
className="p-2 rounded text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
|
||||||
|
title="Logout"
|
||||||
|
>
|
||||||
|
<LogOut className="w-4 h-4" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
);
|
||||||
|
}
|
||||||
238
frontend/src/components/MainDataFeed.tsx
Normal file
238
frontend/src/components/MainDataFeed.tsx
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { useTargetStore } from '../stores/targetStore';
|
||||||
|
import { useScraperStore } from '../stores/scraperStore';
|
||||||
|
import {
|
||||||
|
Globe, Twitter, Instagram, Linkedin, Facebook,
|
||||||
|
ExternalLink, Play, Trash2, ChevronDown, ChevronUp,
|
||||||
|
Database, FileJson, Copy, CheckCircle, AlertCircle
|
||||||
|
} from 'lucide-react';
|
||||||
|
import { formatDistanceToNow } from 'date-fns';
|
||||||
|
|
||||||
|
const PLATFORM_ICONS: Record<string, any> = {
|
||||||
|
twitter: Twitter,
|
||||||
|
x: Twitter,
|
||||||
|
instagram: Instagram,
|
||||||
|
linkedin: Linkedin,
|
||||||
|
facebook: Facebook,
|
||||||
|
default: Globe,
|
||||||
|
};
|
||||||
|
|
||||||
|
const PLATFORM_COLORS: Record<string, string> = {
|
||||||
|
twitter: 'text-sky-400',
|
||||||
|
x: 'text-slate-300',
|
||||||
|
instagram: 'text-pink-400',
|
||||||
|
linkedin: 'text-blue-400',
|
||||||
|
facebook: 'text-blue-500',
|
||||||
|
default: 'text-slate-400',
|
||||||
|
};
|
||||||
|
|
||||||
|
export function MainDataFeed() {
|
||||||
|
const { selectedTarget, deleteProfile } = useTargetStore();
|
||||||
|
const { startJob } = useScraperStore();
|
||||||
|
const [expandedProfile, setExpandedProfile] = useState<string | null>(null);
|
||||||
|
const [scraping, setScraping] = useState<string | null>(null);
|
||||||
|
const [copied, setCopied] = useState<string | null>(null);
|
||||||
|
|
||||||
|
if (!selectedTarget) {
|
||||||
|
return (
|
||||||
|
<div className="h-full flex items-center justify-center bg-slate-950">
|
||||||
|
<div className="text-center">
|
||||||
|
<Database className="w-16 h-16 text-slate-800 mx-auto mb-4" />
|
||||||
|
<h2 className="text-lg font-medium text-slate-400 mb-2">No Target Selected</h2>
|
||||||
|
<p className="text-sm text-slate-600 max-w-xs">
|
||||||
|
Select a target from the left panel to view their social media profiles and scraped data.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const profiles = selectedTarget.profiles || [];
|
||||||
|
|
||||||
|
const handleScrape = async (profileId: string, platform: string, profileUrl?: string) => {
|
||||||
|
setScraping(profileId);
|
||||||
|
try {
|
||||||
|
await startJob({
|
||||||
|
platform,
|
||||||
|
profileUrl,
|
||||||
|
targetId: selectedTarget.id,
|
||||||
|
profileId,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to start scrape:', err);
|
||||||
|
} finally {
|
||||||
|
setScraping(null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleDelete = async (profileId: string) => {
|
||||||
|
if (confirm('Delete this profile?')) {
|
||||||
|
await deleteProfile(selectedTarget.id, profileId);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const copyData = (data: any, id: string) => {
|
||||||
|
navigator.clipboard.writeText(JSON.stringify(data, null, 2));
|
||||||
|
setCopied(id);
|
||||||
|
setTimeout(() => setCopied(null), 2000);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="h-full flex flex-col bg-slate-950">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="p-4 border-b border-slate-800">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div>
|
||||||
|
<h2 className="text-lg font-semibold text-slate-100">{selectedTarget.name}</h2>
|
||||||
|
<p className="text-xs text-slate-500 font-mono mt-0.5">
|
||||||
|
ID: {selectedTarget.id.slice(0, 8)}...
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="text-xs text-slate-500">
|
||||||
|
{profiles.length} profile{profiles.length !== 1 ? 's' : ''}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{selectedTarget.notes && (
|
||||||
|
<p className="text-sm text-slate-400 mt-3 p-3 bg-slate-900 rounded border border-slate-800">
|
||||||
|
{selectedTarget.notes}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Profiles */}
|
||||||
|
<div className="flex-1 overflow-y-auto p-4">
|
||||||
|
{profiles.length === 0 ? (
|
||||||
|
<div className="text-center py-12">
|
||||||
|
<Globe className="w-12 h-12 text-slate-700 mx-auto mb-3" />
|
||||||
|
<p className="text-sm text-slate-500">No profiles linked</p>
|
||||||
|
<p className="text-xs text-slate-600 mt-1">Add social media profiles to start collecting data</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="space-y-3">
|
||||||
|
{profiles.map(profile => {
|
||||||
|
const Icon = PLATFORM_ICONS[profile.platform.toLowerCase()] || PLATFORM_ICONS.default;
|
||||||
|
const colorClass = PLATFORM_COLORS[profile.platform.toLowerCase()] || PLATFORM_COLORS.default;
|
||||||
|
const isExpanded = expandedProfile === profile.id;
|
||||||
|
const profileData = profile.profile_data ? JSON.parse(profile.profile_data) : null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div key={profile.id} className="card border-slate-800">
|
||||||
|
{/* Profile header */}
|
||||||
|
<div className="p-4">
|
||||||
|
<div className="flex items-start justify-between">
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
<div className={`p-2 rounded-lg bg-slate-800 ${colorClass}`}>
|
||||||
|
<Icon className="w-5 h-5" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="font-medium text-slate-200">
|
||||||
|
{profile.username || profile.platform}
|
||||||
|
</span>
|
||||||
|
<span className="text-xs text-slate-600 font-mono uppercase">
|
||||||
|
{profile.platform}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{profile.profile_url && (
|
||||||
|
<a
|
||||||
|
href={profile.profile_url}
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="text-xs text-slate-500 hover:text-accent-cyan flex items-center gap-1 mt-0.5"
|
||||||
|
>
|
||||||
|
<span className="truncate max-w-[200px]">{profile.profile_url}</span>
|
||||||
|
<ExternalLink className="w-3 h-3" />
|
||||||
|
</a>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center gap-1">
|
||||||
|
<button
|
||||||
|
onClick={() => handleScrape(profile.id, profile.platform, profile.profile_url || undefined)}
|
||||||
|
disabled={scraping === profile.id}
|
||||||
|
className="p-2 rounded hover:bg-slate-800 text-accent-cyan transition-colors disabled:opacity-50"
|
||||||
|
title="Start scrape"
|
||||||
|
>
|
||||||
|
{scraping === profile.id ? (
|
||||||
|
<div className="w-4 h-4 border-2 border-accent-cyan border-t-transparent rounded-full animate-spin" />
|
||||||
|
) : (
|
||||||
|
<Play className="w-4 h-4" />
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => handleDelete(profile.id)}
|
||||||
|
className="p-2 rounded hover:bg-slate-800 text-slate-500 hover:text-accent-rose transition-colors"
|
||||||
|
title="Delete profile"
|
||||||
|
>
|
||||||
|
<Trash2 className="w-4 h-4" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Scraped status */}
|
||||||
|
<div className="flex items-center gap-4 mt-3 pt-3 border-t border-slate-800">
|
||||||
|
{profile.last_scraped ? (
|
||||||
|
<div className="flex items-center gap-1.5 text-xs text-accent-emerald">
|
||||||
|
<CheckCircle className="w-3.5 h-3.5" />
|
||||||
|
<span>Last scraped {formatDistanceToNow(new Date(profile.last_scraped), { addSuffix: true })}</span>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="flex items-center gap-1.5 text-xs text-slate-500">
|
||||||
|
<AlertCircle className="w-3.5 h-3.5" />
|
||||||
|
<span>Never scraped</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{profileData && (
|
||||||
|
<button
|
||||||
|
onClick={() => setExpandedProfile(isExpanded ? null : profile.id)}
|
||||||
|
className="flex items-center gap-1 text-xs text-slate-400 hover:text-slate-200"
|
||||||
|
>
|
||||||
|
<FileJson className="w-3.5 h-3.5" />
|
||||||
|
<span>View Data</span>
|
||||||
|
{isExpanded ? <ChevronUp className="w-3.5 h-3.5" /> : <ChevronDown className="w-3.5 h-3.5" />}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Expanded data view */}
|
||||||
|
{isExpanded && profileData && (
|
||||||
|
<div className="border-t border-slate-800 p-4 bg-slate-900/50">
|
||||||
|
<div className="flex items-center justify-between mb-3">
|
||||||
|
<span className="text-xs font-medium text-slate-400">Scraped Data</span>
|
||||||
|
<button
|
||||||
|
onClick={() => copyData(profileData, profile.id)}
|
||||||
|
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300"
|
||||||
|
>
|
||||||
|
{copied === profile.id ? (
|
||||||
|
<>
|
||||||
|
<CheckCircle className="w-3.5 h-3.5 text-accent-emerald" />
|
||||||
|
<span className="text-accent-emerald">Copied!</span>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Copy className="w-3.5 h-3.5" />
|
||||||
|
<span>Copy JSON</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<pre className="text-xs text-slate-300 font-mono bg-slate-950 p-3 rounded overflow-x-auto max-h-64 overflow-y-auto">
|
||||||
|
{JSON.stringify(profileData, null, 2)}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
288
frontend/src/components/ScraperPanel.tsx
Normal file
288
frontend/src/components/ScraperPanel.tsx
Normal file
@@ -0,0 +1,288 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { useScraperStore, ScraperJob, Session } from '../stores/scraperStore';
|
||||||
|
import {
|
||||||
|
Activity, Key, Clock, CheckCircle, XCircle,
|
||||||
|
StopCircle, ChevronDown, Terminal, RefreshCw
|
||||||
|
} from 'lucide-react';
|
||||||
|
import { formatDistanceToNow, format } from 'date-fns';
|
||||||
|
|
||||||
|
type Tab = 'jobs' | 'sessions';
|
||||||
|
|
||||||
|
export function ScraperPanel() {
|
||||||
|
const [activeTab, setActiveTab] = useState<Tab>('jobs');
|
||||||
|
const { jobs, sessions, activeJobs, fetchJobs, fetchSessions } = useScraperStore();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="h-full flex flex-col bg-slate-900/50">
|
||||||
|
{/* Tabs */}
|
||||||
|
<div className="flex border-b border-slate-800">
|
||||||
|
<button
|
||||||
|
onClick={() => setActiveTab('jobs')}
|
||||||
|
className={`flex-1 px-4 py-3 text-xs font-medium flex items-center justify-center gap-2 transition-colors ${
|
||||||
|
activeTab === 'jobs'
|
||||||
|
? 'text-accent-cyan border-b-2 border-accent-cyan bg-slate-800/30'
|
||||||
|
: 'text-slate-400 hover:text-slate-200'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<Activity className="w-3.5 h-3.5" />
|
||||||
|
Jobs
|
||||||
|
{activeJobs.length > 0 && (
|
||||||
|
<span className="px-1.5 py-0.5 rounded-full bg-accent-cyan/20 text-accent-cyan text-[10px]">
|
||||||
|
{activeJobs.length}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => setActiveTab('sessions')}
|
||||||
|
className={`flex-1 px-4 py-3 text-xs font-medium flex items-center justify-center gap-2 transition-colors ${
|
||||||
|
activeTab === 'sessions'
|
||||||
|
? 'text-accent-cyan border-b-2 border-accent-cyan bg-slate-800/30'
|
||||||
|
: 'text-slate-400 hover:text-slate-200'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<Key className="w-3.5 h-3.5" />
|
||||||
|
Sessions
|
||||||
|
<span className="px-1.5 py-0.5 rounded-full bg-slate-700 text-slate-300 text-[10px]">
|
||||||
|
{sessions.length}
|
||||||
|
</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Content */}
|
||||||
|
<div className="flex-1 overflow-y-auto">
|
||||||
|
{activeTab === 'jobs' ? (
|
||||||
|
<JobsList jobs={jobs} onRefresh={fetchJobs} />
|
||||||
|
) : (
|
||||||
|
<SessionsList sessions={sessions} onRefresh={fetchSessions} />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function JobsList({ jobs, onRefresh }: { jobs: ScraperJob[]; onRefresh: () => void }) {
|
||||||
|
const { cancelJob, logs } = useScraperStore();
|
||||||
|
const [expandedJob, setExpandedJob] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const getStatusIcon = (status: string) => {
|
||||||
|
switch (status) {
|
||||||
|
case 'running':
|
||||||
|
return <div className="w-3 h-3 border-2 border-accent-cyan border-t-transparent rounded-full animate-spin" />;
|
||||||
|
case 'completed':
|
||||||
|
return <CheckCircle className="w-4 h-4 text-accent-emerald" />;
|
||||||
|
case 'failed':
|
||||||
|
return <XCircle className="w-4 h-4 text-accent-rose" />;
|
||||||
|
case 'cancelled':
|
||||||
|
return <StopCircle className="w-4 h-4 text-slate-500" />;
|
||||||
|
default:
|
||||||
|
return <Clock className="w-4 h-4 text-accent-amber" />;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const getStatusColor = (status: string) => {
|
||||||
|
switch (status) {
|
||||||
|
case 'running': return 'border-accent-cyan/30 bg-accent-cyan/5';
|
||||||
|
case 'completed': return 'border-accent-emerald/30 bg-accent-emerald/5';
|
||||||
|
case 'failed': return 'border-accent-rose/30 bg-accent-rose/5';
|
||||||
|
default: return 'border-slate-700 bg-slate-800/50';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="p-3 space-y-2">
|
||||||
|
<div className="flex items-center justify-between mb-2">
|
||||||
|
<span className="text-xs text-slate-500">{jobs.length} total jobs</span>
|
||||||
|
<button
|
||||||
|
onClick={onRefresh}
|
||||||
|
className="p-1.5 rounded text-slate-400 hover:text-slate-200 hover:bg-slate-800"
|
||||||
|
>
|
||||||
|
<RefreshCw className="w-3.5 h-3.5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{jobs.length === 0 ? (
|
||||||
|
<div className="text-center py-8">
|
||||||
|
<Activity className="w-10 h-10 text-slate-700 mx-auto mb-2" />
|
||||||
|
<p className="text-xs text-slate-500">No scraper jobs yet</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
jobs.slice(0, 20).map(job => {
|
||||||
|
const isExpanded = expandedJob === job.id;
|
||||||
|
const jobLogs = logs.get(job.id) || [];
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={job.id}
|
||||||
|
className={`rounded-lg border ${getStatusColor(job.status)} transition-colors`}
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
onClick={() => setExpandedJob(isExpanded ? null : job.id)}
|
||||||
|
className="w-full p-3 text-left"
|
||||||
|
>
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{getStatusIcon(job.status)}
|
||||||
|
<span className="text-sm font-medium text-slate-200 capitalize">
|
||||||
|
{job.platform}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<ChevronDown className={`w-4 h-4 text-slate-500 transition-transform ${isExpanded ? 'rotate-180' : ''}`} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center gap-3 mt-2">
|
||||||
|
{job.status === 'running' && (
|
||||||
|
<div className="flex-1">
|
||||||
|
<div className="h-1 bg-slate-700 rounded-full overflow-hidden">
|
||||||
|
<div
|
||||||
|
className="h-full bg-accent-cyan transition-all"
|
||||||
|
style={{ width: `${job.progress}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<span className="text-[10px] text-slate-500 font-mono">
|
||||||
|
{formatDistanceToNow(new Date(job.created_at), { addSuffix: true })}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{isExpanded && (
|
||||||
|
<div className="px-3 pb-3 border-t border-slate-700/50 mt-2 pt-2">
|
||||||
|
<div className="space-y-1 text-xs">
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span className="text-slate-500">Job ID:</span>
|
||||||
|
<span className="text-slate-300 font-mono">{job.id.slice(0, 8)}...</span>
|
||||||
|
</div>
|
||||||
|
{job.target_name && (
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span className="text-slate-500">Target:</span>
|
||||||
|
<span className="text-slate-300">{job.target_name}</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{job.error && (
|
||||||
|
<div className="mt-2 p-2 bg-accent-rose/10 rounded text-accent-rose text-xs">
|
||||||
|
{job.error}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{job.status === 'running' && (
|
||||||
|
<button
|
||||||
|
onClick={() => cancelJob(job.id)}
|
||||||
|
className="w-full mt-2 py-1.5 rounded border border-accent-rose/30 text-accent-rose text-xs hover:bg-accent-rose/10"
|
||||||
|
>
|
||||||
|
Cancel Job
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Logs */}
|
||||||
|
{jobLogs.length > 0 && (
|
||||||
|
<div className="mt-3">
|
||||||
|
<div className="flex items-center gap-1 text-slate-500 mb-2">
|
||||||
|
<Terminal className="w-3 h-3" />
|
||||||
|
<span>Logs</span>
|
||||||
|
</div>
|
||||||
|
<div className="bg-slate-950 rounded p-2 max-h-32 overflow-y-auto font-mono text-[10px] space-y-0.5">
|
||||||
|
{jobLogs.slice(-10).map((log, i) => (
|
||||||
|
<div key={i} className={`${log.level === 'error' ? 'text-accent-rose' : 'text-slate-400'}`}>
|
||||||
|
<span className="text-slate-600">[{format(new Date(log.timestamp), 'HH:mm:ss')}]</span>{' '}
|
||||||
|
{log.message}
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function SessionsList({ sessions, onRefresh }: { sessions: Session[]; onRefresh: () => void }) {
|
||||||
|
const { deleteSession } = useScraperStore();
|
||||||
|
|
||||||
|
const getStatusIndicator = (status: string) => {
|
||||||
|
switch (status) {
|
||||||
|
case 'active':
|
||||||
|
return <div className="status-indicator status-active" />;
|
||||||
|
case 'expired':
|
||||||
|
return <div className="status-indicator status-warning" />;
|
||||||
|
default:
|
||||||
|
return <div className="status-indicator status-error" />;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="p-3 space-y-2">
|
||||||
|
<div className="flex items-center justify-between mb-2">
|
||||||
|
<span className="text-xs text-slate-500">Session Vault</span>
|
||||||
|
<button
|
||||||
|
onClick={onRefresh}
|
||||||
|
className="p-1.5 rounded text-slate-400 hover:text-slate-200 hover:bg-slate-800"
|
||||||
|
>
|
||||||
|
<RefreshCw className="w-3.5 h-3.5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{sessions.length === 0 ? (
|
||||||
|
<div className="text-center py-8">
|
||||||
|
<Key className="w-10 h-10 text-slate-700 mx-auto mb-2" />
|
||||||
|
<p className="text-xs text-slate-500">No sessions stored</p>
|
||||||
|
<p className="text-[10px] text-slate-600 mt-1">Add session cookies to enable authenticated scraping</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
sessions.map(session => (
|
||||||
|
<div key={session.id} className="p-3 rounded-lg border border-slate-800 bg-slate-800/30">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{getStatusIndicator(session.status)}
|
||||||
|
<span className="text-sm font-medium text-slate-200 capitalize">
|
||||||
|
{session.platform}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<span className={`text-[10px] px-2 py-0.5 rounded-full ${
|
||||||
|
session.status === 'active'
|
||||||
|
? 'bg-accent-emerald/20 text-accent-emerald'
|
||||||
|
: session.status === 'expired'
|
||||||
|
? 'bg-accent-amber/20 text-accent-amber'
|
||||||
|
: 'bg-accent-rose/20 text-accent-rose'
|
||||||
|
}`}>
|
||||||
|
{session.status}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="mt-2 space-y-1 text-xs">
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span className="text-slate-500">Name:</span>
|
||||||
|
<span className="text-slate-300">{session.session_name}</span>
|
||||||
|
</div>
|
||||||
|
{session.last_validated && (
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span className="text-slate-500">Validated:</span>
|
||||||
|
<span className="text-slate-400 font-mono text-[10px]">
|
||||||
|
{formatDistanceToNow(new Date(session.last_validated), { addSuffix: true })}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button
|
||||||
|
onClick={() => {
|
||||||
|
if (confirm(`Delete ${session.platform} session?`)) {
|
||||||
|
deleteSession(session.id);
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
className="w-full mt-3 py-1.5 rounded border border-slate-700 text-slate-400 text-xs hover:border-accent-rose/30 hover:text-accent-rose hover:bg-accent-rose/5 transition-colors"
|
||||||
|
>
|
||||||
|
Remove Session
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
))
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
199
frontend/src/components/TargetList.tsx
Normal file
199
frontend/src/components/TargetList.tsx
Normal file
@@ -0,0 +1,199 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import { useTargetStore, Target } from '../stores/targetStore';
|
||||||
|
import {
|
||||||
|
Users, Search, ChevronRight, Plus, MoreHorizontal,
|
||||||
|
Trash2, Edit2, UserCircle2
|
||||||
|
} from 'lucide-react';
|
||||||
|
import { formatDistanceToNow } from 'date-fns';
|
||||||
|
|
||||||
|
interface TargetListProps {
|
||||||
|
onAddProfile: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function TargetList({ onAddProfile }: TargetListProps) {
|
||||||
|
const { targets, selectedTarget, selectTarget, fetchTarget, loading } = useTargetStore();
|
||||||
|
const [search, setSearch] = useState('');
|
||||||
|
const [contextMenu, setContextMenu] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const filteredTargets = targets.filter(t =>
|
||||||
|
t.name.toLowerCase().includes(search.toLowerCase())
|
||||||
|
);
|
||||||
|
|
||||||
|
const handleSelectTarget = async (target: Target) => {
|
||||||
|
selectTarget(target);
|
||||||
|
await fetchTarget(target.id);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="h-full flex flex-col bg-slate-900/50">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="p-3 border-b border-slate-800">
|
||||||
|
<div className="flex items-center justify-between mb-3">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Users className="w-4 h-4 text-slate-400" />
|
||||||
|
<span className="text-sm font-medium text-slate-300">Targets</span>
|
||||||
|
</div>
|
||||||
|
<span className="text-xs text-slate-500 font-mono">{targets.length}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="relative">
|
||||||
|
<Search className="absolute left-2.5 top-1/2 -translate-y-1/2 w-3.5 h-3.5 text-slate-500" />
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
placeholder="Search targets..."
|
||||||
|
value={search}
|
||||||
|
onChange={(e) => setSearch(e.target.value)}
|
||||||
|
className="input-field w-full pl-8 py-1.5 text-xs"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Target list */}
|
||||||
|
<div className="flex-1 overflow-y-auto">
|
||||||
|
{loading && targets.length === 0 ? (
|
||||||
|
<div className="p-4 space-y-2">
|
||||||
|
{[1, 2, 3].map(i => (
|
||||||
|
<div key={i} className="h-16 shimmer rounded" />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
) : filteredTargets.length === 0 ? (
|
||||||
|
<div className="p-4 text-center">
|
||||||
|
<UserCircle2 className="w-10 h-10 text-slate-700 mx-auto mb-2" />
|
||||||
|
<p className="text-sm text-slate-500">
|
||||||
|
{search ? 'No targets found' : 'No targets yet'}
|
||||||
|
</p>
|
||||||
|
<p className="text-xs text-slate-600 mt-1">
|
||||||
|
Add a target to get started
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="p-2 space-y-1">
|
||||||
|
{filteredTargets.map(target => (
|
||||||
|
<TargetItem
|
||||||
|
key={target.id}
|
||||||
|
target={target}
|
||||||
|
isSelected={selectedTarget?.id === target.id}
|
||||||
|
onSelect={() => handleSelectTarget(target)}
|
||||||
|
showContextMenu={contextMenu === target.id}
|
||||||
|
onToggleContext={() => setContextMenu(contextMenu === target.id ? null : target.id)}
|
||||||
|
onAddProfile={onAddProfile}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TargetItemProps {
|
||||||
|
target: Target;
|
||||||
|
isSelected: boolean;
|
||||||
|
onSelect: () => void;
|
||||||
|
showContextMenu: boolean;
|
||||||
|
onToggleContext: () => void;
|
||||||
|
onAddProfile: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function TargetItem({
|
||||||
|
target,
|
||||||
|
isSelected,
|
||||||
|
onSelect,
|
||||||
|
showContextMenu,
|
||||||
|
onToggleContext,
|
||||||
|
onAddProfile
|
||||||
|
}: TargetItemProps) {
|
||||||
|
const { deleteTarget } = useTargetStore();
|
||||||
|
|
||||||
|
const handleDelete = async (e: React.MouseEvent) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
if (confirm(`Delete target "${target.name}"?`)) {
|
||||||
|
await deleteTarget(target.id);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="relative">
|
||||||
|
<button
|
||||||
|
onClick={onSelect}
|
||||||
|
className={`w-full text-left p-3 rounded-lg transition-all ${
|
||||||
|
isSelected
|
||||||
|
? 'bg-slate-800 border border-accent-cyan/30'
|
||||||
|
: 'hover:bg-slate-800/50 border border-transparent'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<div className="flex items-start justify-between">
|
||||||
|
<div className="flex-1 min-w-0">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="text-sm font-medium text-slate-200 truncate">
|
||||||
|
{target.name}
|
||||||
|
</span>
|
||||||
|
{isSelected && (
|
||||||
|
<ChevronRight className="w-3.5 h-3.5 text-accent-cyan flex-shrink-0" />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2 mt-1">
|
||||||
|
<span className="text-xs text-slate-500">
|
||||||
|
{target.profile_count || 0} profiles
|
||||||
|
</span>
|
||||||
|
<span className="text-xs text-slate-600">•</span>
|
||||||
|
<span className="text-[10px] text-slate-600 font-mono">
|
||||||
|
{target.updated_at && !isNaN(new Date(target.updated_at).getTime())
|
||||||
|
? formatDistanceToNow(new Date(target.updated_at), { addSuffix: true })
|
||||||
|
: 'just now'}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
onToggleContext();
|
||||||
|
}}
|
||||||
|
className="p-1 rounded hover:bg-slate-700 text-slate-500 hover:text-slate-300"
|
||||||
|
>
|
||||||
|
<MoreHorizontal className="w-4 h-4" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{/* Context menu */}
|
||||||
|
{showContextMenu && (
|
||||||
|
<>
|
||||||
|
<div
|
||||||
|
className="fixed inset-0 z-10"
|
||||||
|
onClick={onToggleContext}
|
||||||
|
/>
|
||||||
|
<div className="absolute right-2 top-10 z-20 w-40 py-1 bg-slate-800 border border-slate-700 rounded-lg shadow-xl">
|
||||||
|
<button
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
onSelect();
|
||||||
|
onAddProfile();
|
||||||
|
onToggleContext();
|
||||||
|
}}
|
||||||
|
className="w-full px-3 py-2 text-left text-xs text-slate-300 hover:bg-slate-700 flex items-center gap-2"
|
||||||
|
>
|
||||||
|
<Plus className="w-3.5 h-3.5" />
|
||||||
|
Add Profile
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className="w-full px-3 py-2 text-left text-xs text-slate-300 hover:bg-slate-700 flex items-center gap-2"
|
||||||
|
>
|
||||||
|
<Edit2 className="w-3.5 h-3.5" />
|
||||||
|
Edit Target
|
||||||
|
</button>
|
||||||
|
<div className="my-1 border-t border-slate-700" />
|
||||||
|
<button
|
||||||
|
onClick={handleDelete}
|
||||||
|
className="w-full px-3 py-2 text-left text-xs text-accent-rose hover:bg-accent-rose/10 flex items-center gap-2"
|
||||||
|
>
|
||||||
|
<Trash2 className="w-3.5 h-3.5" />
|
||||||
|
Delete Target
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
125
frontend/src/index.css
Normal file
125
frontend/src/index.css
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
@tailwind base;
|
||||||
|
@tailwind components;
|
||||||
|
@tailwind utilities;
|
||||||
|
|
||||||
|
/* Base styles */
|
||||||
|
:root {
|
||||||
|
--color-bg-primary: #020617;
|
||||||
|
--color-bg-secondary: #0f172a;
|
||||||
|
--color-bg-tertiary: #1e293b;
|
||||||
|
--color-text-primary: #f1f5f9;
|
||||||
|
--color-text-secondary: #94a3b8;
|
||||||
|
--color-accent: #22d3ee;
|
||||||
|
}
|
||||||
|
|
||||||
|
html, body, #root {
|
||||||
|
height: 100%;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
font-family: 'Inter', system-ui, sans-serif;
|
||||||
|
background-color: var(--color-bg-primary);
|
||||||
|
color: var(--color-text-primary);
|
||||||
|
-webkit-font-smoothing: antialiased;
|
||||||
|
-moz-osx-font-smoothing: grayscale;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Scrollbar styling */
|
||||||
|
::-webkit-scrollbar {
|
||||||
|
width: 6px;
|
||||||
|
height: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar-track {
|
||||||
|
background: #0f172a;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar-thumb {
|
||||||
|
background: #334155;
|
||||||
|
border-radius: 3px;
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar-thumb:hover {
|
||||||
|
background: #475569;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Custom utilities */
|
||||||
|
@layer utilities {
|
||||||
|
.text-gradient {
|
||||||
|
@apply bg-gradient-to-r from-accent-cyan to-accent-emerald bg-clip-text text-transparent;
|
||||||
|
}
|
||||||
|
|
||||||
|
.border-glow {
|
||||||
|
box-shadow: 0 0 10px rgba(34, 211, 238, 0.2);
|
||||||
|
}
|
||||||
|
|
||||||
|
.card {
|
||||||
|
@apply bg-slate-900 border border-slate-800 rounded-lg;
|
||||||
|
}
|
||||||
|
|
||||||
|
.card-hover {
|
||||||
|
@apply hover:border-slate-700 hover:bg-slate-800/50 transition-all duration-200;
|
||||||
|
}
|
||||||
|
|
||||||
|
.input-field {
|
||||||
|
@apply bg-slate-900 border border-slate-700 rounded px-3 py-2 text-sm text-slate-100
|
||||||
|
placeholder:text-slate-500 focus:outline-none focus:border-accent-cyan focus:ring-1 focus:ring-accent-cyan/30;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-primary {
|
||||||
|
@apply bg-accent-cyan text-slate-950 px-4 py-2 rounded font-medium text-sm
|
||||||
|
hover:bg-accent-cyan/90 active:bg-accent-cyan/80 transition-colors
|
||||||
|
disabled:opacity-50 disabled:cursor-not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-secondary {
|
||||||
|
@apply bg-slate-800 text-slate-200 px-4 py-2 rounded font-medium text-sm border border-slate-700
|
||||||
|
hover:bg-slate-700 hover:border-slate-600 active:bg-slate-600 transition-colors
|
||||||
|
disabled:opacity-50 disabled:cursor-not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-danger {
|
||||||
|
@apply bg-accent-rose/10 text-accent-rose px-4 py-2 rounded font-medium text-sm border border-accent-rose/30
|
||||||
|
hover:bg-accent-rose/20 active:bg-accent-rose/30 transition-colors
|
||||||
|
disabled:opacity-50 disabled:cursor-not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-indicator {
|
||||||
|
@apply w-2 h-2 rounded-full;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-active {
|
||||||
|
@apply bg-accent-emerald animate-pulse;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-warning {
|
||||||
|
@apply bg-accent-amber;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-error {
|
||||||
|
@apply bg-accent-rose;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-inactive {
|
||||||
|
@apply bg-slate-600;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Monospace text */
|
||||||
|
.font-mono {
|
||||||
|
font-family: 'JetBrains Mono', 'Fira Code', monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Animation for loading states */
|
||||||
|
@keyframes shimmer {
|
||||||
|
0% { background-position: -200% 0; }
|
||||||
|
100% { background-position: 200% 0; }
|
||||||
|
}
|
||||||
|
|
||||||
|
.shimmer {
|
||||||
|
background: linear-gradient(90deg, #1e293b 25%, #334155 50%, #1e293b 75%);
|
||||||
|
background-size: 200% 100%;
|
||||||
|
animation: shimmer 1.5s infinite;
|
||||||
|
}
|
||||||
10
frontend/src/main.tsx
Normal file
10
frontend/src/main.tsx
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
import React from 'react'
|
||||||
|
import ReactDOM from 'react-dom/client'
|
||||||
|
import App from './App'
|
||||||
|
import './index.css'
|
||||||
|
|
||||||
|
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||||
|
<React.StrictMode>
|
||||||
|
<App />
|
||||||
|
</React.StrictMode>,
|
||||||
|
)
|
||||||
81
frontend/src/pages/Dashboard.tsx
Normal file
81
frontend/src/pages/Dashboard.tsx
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { useAuthStore } from '../stores/authStore';
|
||||||
|
import { useTargetStore } from '../stores/targetStore';
|
||||||
|
import { useScraperStore } from '../stores/scraperStore';
|
||||||
|
import { TargetList } from '../components/TargetList';
|
||||||
|
import { MainDataFeed } from '../components/MainDataFeed';
|
||||||
|
import { ScraperPanel } from '../components/ScraperPanel';
|
||||||
|
import { Header } from '../components/Header';
|
||||||
|
import { AddTargetModal } from '../components/AddTargetModal';
|
||||||
|
import { AddProfileModal } from '../components/AddProfileModal';
|
||||||
|
import { AddSessionModal } from '../components/AddSessionModal';
|
||||||
|
|
||||||
|
export function Dashboard() {
|
||||||
|
const { checkAuth } = useAuthStore();
|
||||||
|
const { fetchTargets } = useTargetStore();
|
||||||
|
const { fetchJobs, fetchSessions, initSocket, disconnectSocket } = useScraperStore();
|
||||||
|
|
||||||
|
const [showAddTarget, setShowAddTarget] = useState(false);
|
||||||
|
const [showAddProfile, setShowAddProfile] = useState(false);
|
||||||
|
const [showAddSession, setShowAddSession] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
// Initialize
|
||||||
|
checkAuth();
|
||||||
|
fetchTargets();
|
||||||
|
fetchJobs();
|
||||||
|
fetchSessions();
|
||||||
|
initSocket();
|
||||||
|
|
||||||
|
// Polling for updates
|
||||||
|
const interval = setInterval(() => {
|
||||||
|
fetchJobs();
|
||||||
|
fetchSessions();
|
||||||
|
}, 10000);
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
clearInterval(interval);
|
||||||
|
disconnectSocket();
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="h-screen flex flex-col overflow-hidden bg-slate-950">
|
||||||
|
<Header
|
||||||
|
onAddTarget={() => setShowAddTarget(true)}
|
||||||
|
onAddSession={() => setShowAddSession(true)}
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Main 3-column layout */}
|
||||||
|
<div className="flex-1 flex overflow-hidden">
|
||||||
|
{/* Left column - Target List */}
|
||||||
|
<aside className="w-72 flex-shrink-0 border-r border-slate-800 overflow-hidden">
|
||||||
|
<TargetList onAddProfile={() => setShowAddProfile(true)} />
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
{/* Center column - Main Data Feed */}
|
||||||
|
<main className="flex-1 overflow-hidden">
|
||||||
|
<MainDataFeed />
|
||||||
|
</main>
|
||||||
|
|
||||||
|
{/* Right column - Scraper Status & Logs */}
|
||||||
|
<aside className="w-80 flex-shrink-0 border-l border-slate-800 overflow-hidden">
|
||||||
|
<ScraperPanel />
|
||||||
|
</aside>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Modals */}
|
||||||
|
{showAddTarget && (
|
||||||
|
<AddTargetModal onClose={() => setShowAddTarget(false)} />
|
||||||
|
)}
|
||||||
|
|
||||||
|
{showAddProfile && (
|
||||||
|
<AddProfileModal onClose={() => setShowAddProfile(false)} />
|
||||||
|
)}
|
||||||
|
|
||||||
|
{showAddSession && (
|
||||||
|
<AddSessionModal onClose={() => setShowAddSession(false)} />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
148
frontend/src/pages/LoginPage.tsx
Normal file
148
frontend/src/pages/LoginPage.tsx
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
import { useState, useEffect } from 'react';
|
||||||
|
import { useNavigate } from 'react-router-dom';
|
||||||
|
import { useAuthStore } from '../stores/authStore';
|
||||||
|
import { Shield, AlertCircle, Lock, Eye, EyeOff } from 'lucide-react';
|
||||||
|
|
||||||
|
export function LoginPage() {
|
||||||
|
const [password, setPassword] = useState('');
|
||||||
|
const [error, setError] = useState('');
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [showPassword, setShowPassword] = useState(false);
|
||||||
|
const navigate = useNavigate();
|
||||||
|
const { login, checkAuth, isAuthenticated } = useAuthStore();
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
// Check if already authenticated
|
||||||
|
checkAuth().then((authenticated) => {
|
||||||
|
if (authenticated) {
|
||||||
|
navigate('/');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (isAuthenticated) {
|
||||||
|
navigate('/');
|
||||||
|
}
|
||||||
|
}, [isAuthenticated, navigate]);
|
||||||
|
|
||||||
|
const handleSubmit = async (e: React.FormEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
setError('');
|
||||||
|
setLoading(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await login(password);
|
||||||
|
navigate('/');
|
||||||
|
} catch (err: any) {
|
||||||
|
setError(err.message || 'Authentication failed');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="min-h-screen flex items-center justify-center bg-slate-950 px-4">
|
||||||
|
{/* Background pattern */}
|
||||||
|
<div className="absolute inset-0 overflow-hidden">
|
||||||
|
<div className="absolute inset-0 bg-gradient-to-br from-slate-950 via-slate-900 to-slate-950" />
|
||||||
|
<div className="absolute inset-0 opacity-30">
|
||||||
|
<div className="absolute top-1/4 left-1/4 w-96 h-96 bg-accent-cyan/5 rounded-full blur-3xl" />
|
||||||
|
<div className="absolute bottom-1/4 right-1/4 w-96 h-96 bg-accent-violet/5 rounded-full blur-3xl" />
|
||||||
|
</div>
|
||||||
|
{/* Grid pattern */}
|
||||||
|
<svg className="absolute inset-0 w-full h-full opacity-[0.03]">
|
||||||
|
<defs>
|
||||||
|
<pattern id="grid" width="40" height="40" patternUnits="userSpaceOnUse">
|
||||||
|
<path d="M 40 0 L 0 0 0 40" fill="none" stroke="currentColor" strokeWidth="1" />
|
||||||
|
</pattern>
|
||||||
|
</defs>
|
||||||
|
<rect width="100%" height="100%" fill="url(#grid)" />
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="relative w-full max-w-md">
|
||||||
|
{/* Logo/Header */}
|
||||||
|
<div className="text-center mb-8">
|
||||||
|
<div className="inline-flex items-center justify-center w-16 h-16 rounded-2xl bg-gradient-to-br from-accent-cyan/20 to-accent-violet/20 border border-accent-cyan/30 mb-4">
|
||||||
|
<Shield className="w-8 h-8 text-accent-cyan" />
|
||||||
|
</div>
|
||||||
|
<h1 className="text-2xl font-bold text-slate-100 mb-1">OSINT Platform</h1>
|
||||||
|
<p className="text-sm text-slate-500">Private Intelligence Automation</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Login Card */}
|
||||||
|
<div className="card p-6 border-slate-800 shadow-xl shadow-black/20">
|
||||||
|
<div className="flex items-center gap-2 mb-6 pb-4 border-b border-slate-800">
|
||||||
|
<Lock className="w-4 h-4 text-slate-500" />
|
||||||
|
<span className="text-sm font-medium text-slate-400">Master Authentication</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form onSubmit={handleSubmit} className="space-y-4">
|
||||||
|
{error && (
|
||||||
|
<div className="flex items-center gap-2 p-3 rounded bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
|
||||||
|
<AlertCircle className="w-4 h-4 flex-shrink-0" />
|
||||||
|
<span>{error}</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label htmlFor="password" className="block text-sm font-medium text-slate-400 mb-2">
|
||||||
|
Master Password
|
||||||
|
</label>
|
||||||
|
<div className="relative">
|
||||||
|
<input
|
||||||
|
id="password"
|
||||||
|
type={showPassword ? 'text' : 'password'}
|
||||||
|
value={password}
|
||||||
|
onChange={(e) => setPassword(e.target.value)}
|
||||||
|
className="input-field w-full pr-10 font-mono"
|
||||||
|
placeholder="Enter your master password"
|
||||||
|
autoFocus
|
||||||
|
disabled={loading}
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setShowPassword(!showPassword)}
|
||||||
|
className="absolute right-3 top-1/2 -translate-y-1/2 text-slate-500 hover:text-slate-300 transition-colors"
|
||||||
|
>
|
||||||
|
{showPassword ? <EyeOff className="w-4 h-4" /> : <Eye className="w-4 h-4" />}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
disabled={!password || loading}
|
||||||
|
className="btn-primary w-full flex items-center justify-center gap-2"
|
||||||
|
>
|
||||||
|
{loading ? (
|
||||||
|
<>
|
||||||
|
<div className="w-4 h-4 border-2 border-slate-950 border-t-transparent rounded-full animate-spin" />
|
||||||
|
<span>Authenticating...</span>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Shield className="w-4 h-4" />
|
||||||
|
<span>Authenticate</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<div className="mt-6 pt-4 border-t border-slate-800">
|
||||||
|
<div className="flex items-center gap-2 text-xs text-slate-600">
|
||||||
|
<div className="w-1.5 h-1.5 rounded-full bg-accent-emerald animate-pulse" />
|
||||||
|
<span>Encrypted connection • Rate limited</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Footer */}
|
||||||
|
<p className="text-center text-xs text-slate-600 mt-6">
|
||||||
|
All access attempts are logged and monitored.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
86
frontend/src/stores/authStore.ts
Normal file
86
frontend/src/stores/authStore.ts
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
import { create } from 'zustand';
|
||||||
|
import { persist } from 'zustand/middleware';
|
||||||
|
|
||||||
|
interface AuthState {
|
||||||
|
isAuthenticated: boolean;
|
||||||
|
token: string | null;
|
||||||
|
login: (password: string) => Promise<boolean>;
|
||||||
|
logout: () => void;
|
||||||
|
checkAuth: () => Promise<boolean>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useAuthStore = create<AuthState>()(
|
||||||
|
persist(
|
||||||
|
(set, get) => ({
|
||||||
|
isAuthenticated: false,
|
||||||
|
token: null,
|
||||||
|
|
||||||
|
login: async (password: string) => {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/auth/login', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ password }),
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const error = await response.json();
|
||||||
|
throw new Error(error.error || 'Login failed');
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await response.json();
|
||||||
|
set({ isAuthenticated: true, token: data.token });
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
set({ isAuthenticated: false, token: null });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
logout: async () => {
|
||||||
|
try {
|
||||||
|
await fetch('/api/auth/logout', {
|
||||||
|
method: 'POST',
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
// Ignore errors
|
||||||
|
}
|
||||||
|
set({ isAuthenticated: false, token: null });
|
||||||
|
},
|
||||||
|
|
||||||
|
checkAuth: async () => {
|
||||||
|
const { token } = get();
|
||||||
|
|
||||||
|
if (!token) {
|
||||||
|
set({ isAuthenticated: false });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/auth/verify', {
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${token}`,
|
||||||
|
},
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
set({ isAuthenticated: true });
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// Token invalid
|
||||||
|
}
|
||||||
|
|
||||||
|
set({ isAuthenticated: false, token: null });
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
name: 'osint-auth',
|
||||||
|
partialize: (state) => ({ token: state.token }),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
206
frontend/src/stores/scraperStore.ts
Normal file
206
frontend/src/stores/scraperStore.ts
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
import { create } from 'zustand';
|
||||||
|
import { io, Socket } from 'socket.io-client';
|
||||||
|
import { useAuthStore } from './authStore';
|
||||||
|
|
||||||
|
export interface ScraperJob {
|
||||||
|
id: string;
|
||||||
|
target_id?: string;
|
||||||
|
profile_id?: string;
|
||||||
|
platform: string;
|
||||||
|
status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
|
||||||
|
progress: number;
|
||||||
|
result?: string;
|
||||||
|
error?: string;
|
||||||
|
target_name?: string;
|
||||||
|
profile_username?: string;
|
||||||
|
started_at?: string;
|
||||||
|
completed_at?: string;
|
||||||
|
created_at: string;
|
||||||
|
logs?: ScraperLog[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ScraperLog {
|
||||||
|
id: number;
|
||||||
|
job_id: string;
|
||||||
|
level: string;
|
||||||
|
message: string;
|
||||||
|
timestamp: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Session {
|
||||||
|
id: string;
|
||||||
|
platform: string;
|
||||||
|
session_name: string;
|
||||||
|
user_agent?: string;
|
||||||
|
proxy?: string;
|
||||||
|
status: 'active' | 'expired' | 'invalid';
|
||||||
|
last_validated?: string;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ScraperState {
|
||||||
|
jobs: ScraperJob[];
|
||||||
|
activeJobs: ScraperJob[];
|
||||||
|
sessions: Session[];
|
||||||
|
logs: Map<string, ScraperLog[]>;
|
||||||
|
socket: Socket | null;
|
||||||
|
loading: boolean;
|
||||||
|
error: string | null;
|
||||||
|
|
||||||
|
initSocket: () => void;
|
||||||
|
disconnectSocket: () => void;
|
||||||
|
fetchJobs: () => Promise<void>;
|
||||||
|
fetchSessions: () => Promise<void>;
|
||||||
|
startJob: (config: { platform: string; profileUrl?: string; targetId?: string; profileId?: string }) => Promise<ScraperJob>;
|
||||||
|
cancelJob: (jobId: string) => Promise<void>;
|
||||||
|
subscribeToJob: (jobId: string) => void;
|
||||||
|
addSession: (data: { platform: string; session_name: string; cookies: any[]; user_agent?: string }) => Promise<void>;
|
||||||
|
deleteSession: (id: string) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useScraperStore = create<ScraperState>((set, get) => ({
|
||||||
|
jobs: [],
|
||||||
|
activeJobs: [],
|
||||||
|
sessions: [],
|
||||||
|
logs: new Map(),
|
||||||
|
socket: null,
|
||||||
|
loading: false,
|
||||||
|
error: null,
|
||||||
|
|
||||||
|
initSocket: () => {
|
||||||
|
const token = useAuthStore.getState().token;
|
||||||
|
if (!token) return;
|
||||||
|
|
||||||
|
const socket = io({
|
||||||
|
auth: { token },
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('connect', () => {
|
||||||
|
console.log('Socket connected');
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('scraper:log', (log: ScraperLog & { jobId: string }) => {
|
||||||
|
const logs = new Map(get().logs);
|
||||||
|
const jobLogs = logs.get(log.jobId) || [];
|
||||||
|
logs.set(log.jobId, [...jobLogs, log]);
|
||||||
|
set({ logs });
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('scraper:status', (update: { jobId: string; status: string; progress?: number }) => {
|
||||||
|
const jobs = get().jobs.map(job =>
|
||||||
|
job.id === update.jobId
|
||||||
|
? { ...job, status: update.status as any, progress: update.progress ?? job.progress }
|
||||||
|
: job
|
||||||
|
);
|
||||||
|
set({ jobs });
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('scraper:jobUpdate', () => {
|
||||||
|
get().fetchJobs();
|
||||||
|
});
|
||||||
|
|
||||||
|
set({ socket });
|
||||||
|
},
|
||||||
|
|
||||||
|
disconnectSocket: () => {
|
||||||
|
const { socket } = get();
|
||||||
|
if (socket) {
|
||||||
|
socket.disconnect();
|
||||||
|
set({ socket: null });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
fetchJobs: async () => {
|
||||||
|
set({ loading: true });
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/scraper/jobs', { credentials: 'include' });
|
||||||
|
if (!response.ok) throw new Error('Failed to fetch jobs');
|
||||||
|
const jobs = await response.json();
|
||||||
|
|
||||||
|
const statusResponse = await fetch('/api/scraper/status', { credentials: 'include' });
|
||||||
|
const statusData = await statusResponse.json();
|
||||||
|
|
||||||
|
set({
|
||||||
|
jobs,
|
||||||
|
activeJobs: statusData.activeJobs || [],
|
||||||
|
loading: false
|
||||||
|
});
|
||||||
|
} catch (error: any) {
|
||||||
|
set({ error: error.message, loading: false });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
fetchSessions: async () => {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/sessions', { credentials: 'include' });
|
||||||
|
if (!response.ok) throw new Error('Failed to fetch sessions');
|
||||||
|
const sessions = await response.json();
|
||||||
|
set({ sessions });
|
||||||
|
} catch (error: any) {
|
||||||
|
set({ error: error.message });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
startJob: async (config) => {
|
||||||
|
const response = await fetch('/api/scraper/start', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
platform: config.platform,
|
||||||
|
profile_url: config.profileUrl,
|
||||||
|
target_id: config.targetId,
|
||||||
|
profile_id: config.profileId,
|
||||||
|
}),
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) throw new Error('Failed to start job');
|
||||||
|
const job = await response.json();
|
||||||
|
|
||||||
|
// Subscribe to job updates
|
||||||
|
get().subscribeToJob(job.id);
|
||||||
|
await get().fetchJobs();
|
||||||
|
|
||||||
|
return job;
|
||||||
|
},
|
||||||
|
|
||||||
|
cancelJob: async (jobId: string) => {
|
||||||
|
const response = await fetch(`/api/scraper/jobs/${jobId}/cancel`, {
|
||||||
|
method: 'POST',
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) throw new Error('Failed to cancel job');
|
||||||
|
await get().fetchJobs();
|
||||||
|
},
|
||||||
|
|
||||||
|
subscribeToJob: (jobId: string) => {
|
||||||
|
const { socket } = get();
|
||||||
|
if (socket) {
|
||||||
|
socket.emit('subscribe:scraper', jobId);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
addSession: async (data) => {
|
||||||
|
const response = await fetch('/api/sessions', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(data),
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) throw new Error('Failed to add session');
|
||||||
|
await get().fetchSessions();
|
||||||
|
},
|
||||||
|
|
||||||
|
deleteSession: async (id: string) => {
|
||||||
|
const response = await fetch(`/api/sessions/${id}`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) throw new Error('Failed to delete session');
|
||||||
|
await get().fetchSessions();
|
||||||
|
},
|
||||||
|
}));
|
||||||
143
frontend/src/stores/targetStore.ts
Normal file
143
frontend/src/stores/targetStore.ts
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
import { create } from 'zustand';
|
||||||
|
|
||||||
|
export interface Target {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
notes?: string;
|
||||||
|
profile_count?: number;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
profiles?: SocialProfile[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SocialProfile {
|
||||||
|
id: string;
|
||||||
|
target_id: string;
|
||||||
|
platform: string;
|
||||||
|
username?: string;
|
||||||
|
profile_url?: string;
|
||||||
|
profile_data?: string;
|
||||||
|
last_scraped?: string;
|
||||||
|
created_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TargetState {
|
||||||
|
targets: Target[];
|
||||||
|
selectedTarget: Target | null;
|
||||||
|
loading: boolean;
|
||||||
|
error: string | null;
|
||||||
|
fetchTargets: () => Promise<void>;
|
||||||
|
fetchTarget: (id: string) => Promise<void>;
|
||||||
|
selectTarget: (target: Target | null) => void;
|
||||||
|
createTarget: (name: string, notes?: string) => Promise<Target>;
|
||||||
|
updateTarget: (id: string, name: string, notes?: string) => Promise<void>;
|
||||||
|
deleteTarget: (id: string) => Promise<void>;
|
||||||
|
addProfile: (targetId: string, platform: string, username?: string, profileUrl?: string) => Promise<void>;
|
||||||
|
deleteProfile: (targetId: string, profileId: string) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useTargetStore = create<TargetState>((set, get) => ({
|
||||||
|
targets: [],
|
||||||
|
selectedTarget: null,
|
||||||
|
loading: false,
|
||||||
|
error: null,
|
||||||
|
|
||||||
|
fetchTargets: async () => {
|
||||||
|
set({ loading: true, error: null });
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/targets', { credentials: 'include' });
|
||||||
|
if (!response.ok) throw new Error('Failed to fetch targets');
|
||||||
|
const data = await response.json();
|
||||||
|
const targets = data.map((t: any) => ({
|
||||||
|
...t,
|
||||||
|
updated_at: t.updatedAt || t.updated_at,
|
||||||
|
created_at: t.createdAt || t.created_at
|
||||||
|
}));
|
||||||
|
set({ targets, loading: false });
|
||||||
|
} catch (error: any) {
|
||||||
|
set({ error: error.message, loading: false });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
fetchTarget: async (id: string) => {
|
||||||
|
set({ loading: true, error: null });
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/targets/${id}`, { credentials: 'include' });
|
||||||
|
if (!response.ok) throw new Error('Failed to fetch target');
|
||||||
|
const t = await response.json();
|
||||||
|
const target = {
|
||||||
|
...t,
|
||||||
|
updated_at: t.updatedAt || t.updated_at,
|
||||||
|
created_at: t.createdAt || t.created_at
|
||||||
|
};
|
||||||
|
set({ selectedTarget: target, loading: false });
|
||||||
|
} catch (error: any) {
|
||||||
|
set({ error: error.message, loading: false });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
selectTarget: (target) => {
|
||||||
|
set({ selectedTarget: target });
|
||||||
|
},
|
||||||
|
|
||||||
|
createTarget: async (name: string, notes?: string) => {
|
||||||
|
const response = await fetch('/api/targets', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ name, notes }),
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error('Failed to create target');
|
||||||
|
const target = await response.json();
|
||||||
|
await get().fetchTargets();
|
||||||
|
return target;
|
||||||
|
},
|
||||||
|
|
||||||
|
updateTarget: async (id: string, name: string, notes?: string) => {
|
||||||
|
const response = await fetch(`/api/targets/${id}`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ name, notes }),
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error('Failed to update target');
|
||||||
|
await get().fetchTargets();
|
||||||
|
if (get().selectedTarget?.id === id) {
|
||||||
|
await get().fetchTarget(id);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
deleteTarget: async (id: string) => {
|
||||||
|
const response = await fetch(`/api/targets/${id}`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error('Failed to delete target');
|
||||||
|
if (get().selectedTarget?.id === id) {
|
||||||
|
set({ selectedTarget: null });
|
||||||
|
}
|
||||||
|
await get().fetchTargets();
|
||||||
|
},
|
||||||
|
|
||||||
|
addProfile: async (targetId: string, platform: string, username?: string, profileUrl?: string) => {
|
||||||
|
const response = await fetch(`/api/targets/${targetId}/profiles`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ platform, username, profile_url: profileUrl }),
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error('Failed to add profile');
|
||||||
|
await get().fetchTarget(targetId);
|
||||||
|
await get().fetchTargets();
|
||||||
|
},
|
||||||
|
|
||||||
|
deleteProfile: async (targetId: string, profileId: string) => {
|
||||||
|
const response = await fetch(`/api/targets/${targetId}/profiles/${profileId}`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
credentials: 'include',
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error('Failed to delete profile');
|
||||||
|
await get().fetchTarget(targetId);
|
||||||
|
await get().fetchTargets();
|
||||||
|
},
|
||||||
|
}));
|
||||||
48
frontend/tailwind.config.js
Normal file
48
frontend/tailwind.config.js
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
/** @type {import('tailwindcss').Config} */
|
||||||
|
export default {
|
||||||
|
content: [
|
||||||
|
"./index.html",
|
||||||
|
"./src/**/*.{js,ts,jsx,tsx}",
|
||||||
|
],
|
||||||
|
theme: {
|
||||||
|
extend: {
|
||||||
|
colors: {
|
||||||
|
// Midnight Utility Theme
|
||||||
|
slate: {
|
||||||
|
950: '#020617',
|
||||||
|
900: '#0f172a',
|
||||||
|
800: '#1e293b',
|
||||||
|
700: '#334155',
|
||||||
|
600: '#475569',
|
||||||
|
500: '#64748b',
|
||||||
|
400: '#94a3b8',
|
||||||
|
300: '#cbd5e1',
|
||||||
|
200: '#e2e8f0',
|
||||||
|
100: '#f1f5f9',
|
||||||
|
},
|
||||||
|
accent: {
|
||||||
|
cyan: '#22d3ee',
|
||||||
|
emerald: '#10b981',
|
||||||
|
amber: '#f59e0b',
|
||||||
|
rose: '#f43f5e',
|
||||||
|
violet: '#8b5cf6',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fontFamily: {
|
||||||
|
sans: ['Inter', 'system-ui', 'sans-serif'],
|
||||||
|
mono: ['JetBrains Mono', 'Fira Code', 'monospace'],
|
||||||
|
},
|
||||||
|
animation: {
|
||||||
|
'pulse-slow': 'pulse 3s cubic-bezier(0.4, 0, 0.6, 1) infinite',
|
||||||
|
'glow': 'glow 2s ease-in-out infinite alternate',
|
||||||
|
},
|
||||||
|
keyframes: {
|
||||||
|
glow: {
|
||||||
|
'0%': { boxShadow: '0 0 5px rgba(34, 211, 238, 0.3)' },
|
||||||
|
'100%': { boxShadow: '0 0 20px rgba(34, 211, 238, 0.6)' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
plugins: [],
|
||||||
|
}
|
||||||
25
frontend/tsconfig.json
Normal file
25
frontend/tsconfig.json
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2020",
|
||||||
|
"useDefineForClassFields": true,
|
||||||
|
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||||
|
"module": "ESNext",
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"allowImportingTsExtensions": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"jsx": "react-jsx",
|
||||||
|
"strict": true,
|
||||||
|
"noUnusedLocals": true,
|
||||||
|
"noUnusedParameters": true,
|
||||||
|
"noFallthroughCasesInSwitch": true,
|
||||||
|
"baseUrl": ".",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["src/*"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"include": ["src"],
|
||||||
|
"references": [{ "path": "./tsconfig.node.json" }]
|
||||||
|
}
|
||||||
11
frontend/tsconfig.node.json
Normal file
11
frontend/tsconfig.node.json
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"composite": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"allowSyntheticDefaultImports": true,
|
||||||
|
"strict": true
|
||||||
|
},
|
||||||
|
"include": ["vite.config.ts"]
|
||||||
|
}
|
||||||
25
frontend/vite.config.ts
Normal file
25
frontend/vite.config.ts
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
import { defineConfig } from 'vite'
|
||||||
|
import react from '@vitejs/plugin-react'
|
||||||
|
|
||||||
|
// https://vitejs.dev/config/
|
||||||
|
export default defineConfig({
|
||||||
|
plugins: [react()],
|
||||||
|
server: {
|
||||||
|
port: 5173,
|
||||||
|
proxy: {
|
||||||
|
'/api': {
|
||||||
|
target: 'http://localhost:3001',
|
||||||
|
changeOrigin: true,
|
||||||
|
},
|
||||||
|
'/socket.io': {
|
||||||
|
target: 'http://localhost:3001',
|
||||||
|
changeOrigin: true,
|
||||||
|
ws: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
build: {
|
||||||
|
outDir: 'dist',
|
||||||
|
sourcemap: false,
|
||||||
|
},
|
||||||
|
})
|
||||||
372
package-lock.json
generated
Normal file
372
package-lock.json
generated
Normal file
@@ -0,0 +1,372 @@
|
|||||||
|
{
|
||||||
|
"name": "osint-platform",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"lockfileVersion": 3,
|
||||||
|
"requires": true,
|
||||||
|
"packages": {
|
||||||
|
"": {
|
||||||
|
"name": "osint-platform",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"devDependencies": {
|
||||||
|
"concurrently": "^8.2.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@babel/runtime": {
|
||||||
|
"version": "7.28.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz",
|
||||||
|
"integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=6.9.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/ansi-regex": {
|
||||||
|
"version": "5.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
|
||||||
|
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/ansi-styles": {
|
||||||
|
"version": "4.3.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||||
|
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"color-convert": "^2.0.1"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=8"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/chalk": {
|
||||||
|
"version": "4.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||||
|
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"ansi-styles": "^4.1.0",
|
||||||
|
"supports-color": "^7.1.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/chalk/node_modules/supports-color": {
|
||||||
|
"version": "7.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||||
|
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"has-flag": "^4.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/cliui": {
|
||||||
|
"version": "8.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
|
||||||
|
"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"string-width": "^4.2.0",
|
||||||
|
"strip-ansi": "^6.0.1",
|
||||||
|
"wrap-ansi": "^7.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/color-convert": {
|
||||||
|
"version": "2.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||||
|
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"color-name": "~1.1.4"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=7.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/color-name": {
|
||||||
|
"version": "1.1.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||||
|
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/concurrently": {
|
||||||
|
"version": "8.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.2.tgz",
|
||||||
|
"integrity": "sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"chalk": "^4.1.2",
|
||||||
|
"date-fns": "^2.30.0",
|
||||||
|
"lodash": "^4.17.21",
|
||||||
|
"rxjs": "^7.8.1",
|
||||||
|
"shell-quote": "^1.8.1",
|
||||||
|
"spawn-command": "0.0.2",
|
||||||
|
"supports-color": "^8.1.1",
|
||||||
|
"tree-kill": "^1.2.2",
|
||||||
|
"yargs": "^17.7.2"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"conc": "dist/bin/concurrently.js",
|
||||||
|
"concurrently": "dist/bin/concurrently.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "^14.13.0 || >=16.0.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/open-cli-tools/concurrently?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/date-fns": {
|
||||||
|
"version": "2.30.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
|
||||||
|
"integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"@babel/runtime": "^7.21.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.11"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "opencollective",
|
||||||
|
"url": "https://opencollective.com/date-fns"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/emoji-regex": {
|
||||||
|
"version": "8.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
|
||||||
|
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/escalade": {
|
||||||
|
"version": "3.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
||||||
|
"integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/get-caller-file": {
|
||||||
|
"version": "2.0.5",
|
||||||
|
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||||
|
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": "6.* || 8.* || >= 10.*"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/has-flag": {
|
||||||
|
"version": "4.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||||
|
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/is-fullwidth-code-point": {
|
||||||
|
"version": "3.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||||
|
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/lodash": {
|
||||||
|
"version": "4.17.23",
|
||||||
|
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz",
|
||||||
|
"integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/require-directory": {
|
||||||
|
"version": "2.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||||
|
"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=0.10.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/rxjs": {
|
||||||
|
"version": "7.8.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
|
||||||
|
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"tslib": "^2.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/shell-quote": {
|
||||||
|
"version": "1.8.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz",
|
||||||
|
"integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 0.4"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/ljharb"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/spawn-command": {
|
||||||
|
"version": "0.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz",
|
||||||
|
"integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"node_modules/string-width": {
|
||||||
|
"version": "4.2.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
|
||||||
|
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"emoji-regex": "^8.0.0",
|
||||||
|
"is-fullwidth-code-point": "^3.0.0",
|
||||||
|
"strip-ansi": "^6.0.1"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/strip-ansi": {
|
||||||
|
"version": "6.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
|
||||||
|
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"ansi-regex": "^5.0.1"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/supports-color": {
|
||||||
|
"version": "8.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
|
||||||
|
"integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"has-flag": "^4.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/supports-color?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/tree-kill": {
|
||||||
|
"version": "1.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
|
||||||
|
"integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"bin": {
|
||||||
|
"tree-kill": "cli.js"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/tslib": {
|
||||||
|
"version": "2.8.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||||
|
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "0BSD"
|
||||||
|
},
|
||||||
|
"node_modules/wrap-ansi": {
|
||||||
|
"version": "7.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||||
|
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"ansi-styles": "^4.0.0",
|
||||||
|
"string-width": "^4.1.0",
|
||||||
|
"strip-ansi": "^6.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/y18n": {
|
||||||
|
"version": "5.0.8",
|
||||||
|
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||||
|
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/yargs": {
|
||||||
|
"version": "17.7.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
|
||||||
|
"integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"cliui": "^8.0.1",
|
||||||
|
"escalade": "^3.1.1",
|
||||||
|
"get-caller-file": "^2.0.5",
|
||||||
|
"require-directory": "^2.1.1",
|
||||||
|
"string-width": "^4.2.3",
|
||||||
|
"y18n": "^5.0.5",
|
||||||
|
"yargs-parser": "^21.1.1"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/yargs-parser": {
|
||||||
|
"version": "21.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
|
||||||
|
"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
19
package.json
Normal file
19
package.json
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"name": "osint-platform",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "Private OSINT Automation Platform",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"dev": "concurrently \"npm run dev:backend\" \"npm run dev:frontend\"",
|
||||||
|
"dev:backend": "cd backend && npm run dev",
|
||||||
|
"dev:frontend": "cd frontend && npm run dev",
|
||||||
|
"build": "npm run build:frontend && npm run build:backend",
|
||||||
|
"build:frontend": "cd frontend && npm run build",
|
||||||
|
"build:backend": "cd backend && npm run build",
|
||||||
|
"start": "cd backend && npm start",
|
||||||
|
"install:all": "npm install && cd backend && npm install && cd ../frontend && npm install"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"concurrently": "^8.2.2"
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user