first commit

This commit is contained in:
2026-01-25 23:05:41 +02:00
commit dec7844b49
48 changed files with 10815 additions and 0 deletions

47
.env.example Normal file
View File

@@ -0,0 +1,47 @@
# ============================================
# OSINT Platform Environment Configuration
# ============================================
# Copy this file to .env and configure your values
# Server Configuration
PORT=3001
NODE_ENV=production
# ===========================================
# DATABASE (MongoDB)
# ===========================================
# MongoDB Connection URI
# For local: mongodb://localhost:27017/osint_platform
# For Atlas: mongodb+srv://user:password@cluster.mongodb.net/osint_platform
MONGODB_URI=mongodb://localhost:27017/osint_platform
# ===========================================
# REQUIRED SECURITY SETTINGS
# ===========================================
# Master Password - CHANGE THIS!
# This is the only password protecting your entire platform
MASTER_PASSWORD=your_secure_master_password_here
# JWT Secret - Generate a random 64-character string
# You can use: openssl rand -hex 32
JWT_SECRET=your_jwt_secret_here_generate_with_openssl_rand_hex_32
# Session Configuration
SESSION_EXPIRY=24h
# ===========================================
# ENCRYPTION SETTINGS
# ===========================================
# Vault Encryption Key - Generate a 64-character hex string (32 bytes)
# You can use: openssl rand -hex 32
VAULT_ENCRYPTION_KEY=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
# ===========================================
# FRONTEND URL (for CORS)
# ===========================================
# Set this to your actual domain in production
FRONTEND_URL=http://localhost:3001

44
.gitignore vendored Normal file
View File

@@ -0,0 +1,44 @@
# Dependencies
node_modules/
# Build outputs
dist/
frontend/dist/
backend/dist/
# Environment files
.env
.env.local
.env.*.local
# Database
*.db
*.db-journal
*.db-wal
*.db-shm
data/
# Logs
logs/
*.log
# IDE
.vscode/
.idea/
*.swp
*.swo
# OS
.DS_Store
Thumbs.db
# Test coverage
coverage/
# Playwright
playwright-report/
test-results/
# Misc
*.pem
*.key

111
Dockerfile Normal file
View File

@@ -0,0 +1,111 @@
# ============================================
# OSINT Platform - Multi-stage Dockerfile
# Optimized for Coolify deployment
# ============================================
# Stage 1: Build frontend
FROM node:20-alpine AS frontend-builder
WORKDIR /app/frontend
# Copy frontend package files
COPY frontend/package*.json ./
# Install dependencies
RUN npm ci --legacy-peer-deps
# Copy frontend source
COPY frontend/ ./
# Build frontend
RUN npm run build
# ============================================
# Stage 2: Build backend
FROM node:20-alpine AS backend-builder
WORKDIR /app/backend
# Copy backend package files
COPY backend/package*.json ./
# Install dependencies (including dev deps for build)
RUN npm ci
# Copy backend source
COPY backend/ ./
# Build TypeScript
RUN npm run build
# ============================================
# Stage 3: Production runtime
FROM node:20-slim AS production
# Install Playwright dependencies
# This includes all browser deps for Chromium
RUN apt-get update && apt-get install -y --no-install-recommends \
# Playwright/Chromium dependencies
libnss3 \
libnspr4 \
libatk1.0-0 \
libatk-bridge2.0-0 \
libcups2 \
libdrm2 \
libdbus-1-3 \
libxkbcommon0 \
libxcomposite1 \
libxdamage1 \
libxfixes3 \
libxrandr2 \
libgbm1 \
libasound2 \
libpango-1.0-0 \
libcairo2 \
libatspi2.0-0 \
# Additional utilities
wget \
ca-certificates \
fonts-liberation \
# Clean up
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean
# Create non-root user for security
RUN groupadd -r osint && useradd -r -g osint osint
WORKDIR /app
# Copy backend production files
COPY --from=backend-builder /app/backend/dist ./dist
COPY --from=backend-builder /app/backend/package*.json ./
# Install production dependencies only
RUN npm ci --only=production --legacy-peer-deps
# Install Playwright browsers (Chromium only for smaller image)
RUN npx playwright install chromium
# Copy frontend build
COPY --from=frontend-builder /app/frontend/dist ./frontend/dist
# Create logs directory with proper permissions
RUN mkdir -p /app/logs && \
chown -R osint:osint /app
# Environment variables
ENV NODE_ENV=production
ENV PORT=3001
# Expose port
EXPOSE 3001
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
CMD wget --no-verbose --tries=1 --spider http://localhost:3001/api/health || exit 1
# Switch to non-root user
USER osint
# Start the application
CMD ["node", "dist/index.js"]

182
README.md Normal file
View File

@@ -0,0 +1,182 @@
# OSINT Automation Platform
A private, containerized OSINT (Open Source Intelligence) automation platform for gathering social media intelligence using authenticated browser sessions.
⚠️ **DISCLAIMER**: This tool is intended for authorized intelligence gathering and research purposes only. Ensure you comply with all applicable laws and platform terms of service.
## Features
- 🔐 **Master Password Protection** - Single secure entry point
- 🎭 **Session Vault** - Store and reuse authenticated social media sessions (encrypted)
- 🤖 **Automated Scraping** - Playwright-based stealth scraping engine
- 📊 **Target Management** - Organize and track multiple investigation targets
- 📱 **Multi-Platform Support** - X/Twitter, Instagram, LinkedIn, Facebook
- 🔒 **Encrypted Storage** - AES-256-GCM encrypted session data
- 📈 **Real-time Progress** - WebSocket-based live job updates
- 🐳 **Docker Ready** - Optimized for Coolify deployment
## Tech Stack
- **Backend**: Node.js, Express, TypeScript
- **Frontend**: React, Vite, TypeScript, Tailwind CSS
- **Database**: MongoDB
- **Browser Automation**: Playwright with stealth plugins
- **Real-time**: Socket.IO
- **State Management**: Zustand
## Quick Start
### Development
1. Clone the repository:
```bash
git clone <repo-url>
cd osint-platform
```
2. Install dependencies:
```bash
npm install
cd backend && npm install
cd ../frontend && npm install
cd ..
```
3. Create environment files:
```bash
# Backend
cp backend/.env.example backend/.env
# Edit backend/.env with your configuration (especially MONGODB_URI)
```
4. Start development servers:
```bash
npm run dev
```
This will start:
- Backend on `http://localhost:3001`
- Frontend on `http://localhost:5173`
### Production (Docker)
1. Create your `.env` file:
```bash
cp .env.example .env
# Edit .env with secure values
```
2. Build and run with Docker Compose:
```bash
docker-compose up -d
```
The application will be available at `http://localhost:3001`
## Coolify Deployment
1. Create a new service in Coolify
2. Point to your Git repository
3. Set Build Pack to "Dockerfile"
4. Configure environment variables:
- `MONGODB_URI` - Your MongoDB connection string (e.g., MongoDB Atlas)
- `MASTER_PASSWORD` - Your secure master password
- `JWT_SECRET` - Generate with `openssl rand -hex 32`
- `VAULT_ENCRYPTION_KEY` - Generate with `openssl rand -hex 32`
- `FRONTEND_URL` - Your domain (e.g., `https://osint.yourdomain.com`)
## Configuration
### Environment Variables
| Variable | Description | Required |
|----------|-------------|----------|
| `MONGODB_URI` | MongoDB connection string | Yes |
| `MASTER_PASSWORD` | Password to access the platform | Yes |
| `JWT_SECRET` | Secret for JWT token signing | Yes |
| `VAULT_ENCRYPTION_KEY` | 64-char hex key for session encryption | Yes |
| `PORT` | Server port (default: 3001) | No |
| `SESSION_EXPIRY` | JWT expiry time (default: 24h) | No |
| `FRONTEND_URL` | Frontend URL for CORS | No |
### Generating Secrets
```bash
# Generate JWT Secret
openssl rand -hex 32
# Generate Vault Encryption Key
openssl rand -hex 32
```
### MongoDB Setup
You can use:
- **MongoDB Atlas** (recommended for production): Create a free cluster at [mongodb.com/atlas](https://mongodb.com/atlas)
- **Local MongoDB**: `mongodb://localhost:27017/osint_platform`
- **Docker MongoDB**: Uncomment the mongodb service in docker-compose.yml
## Usage
### 1. Login
Access the platform and enter your master password.
### 2. Add Sessions
Navigate to "Add Session" and provide:
- Platform (X, Instagram, LinkedIn, Facebook)
- Session name
- Cookies JSON (export from your browser)
**Getting cookies:**
1. Log into the platform in your browser
2. Open DevTools → Application → Cookies
3. Export as JSON using the Cookie Editor extension
### 3. Create Targets
Add investigation targets with optional notes.
### 4. Add Profiles
Link social media profiles to targets:
- Platform
- Username
- Profile URL
### 5. Run Scrapers
Click the play button on any profile to start scraping. Monitor progress in the right panel.
## Architecture
```
osint-platform/
├── backend/
│ ├── src/
│ │ ├── database/ # MongoDB connection
│ │ ├── middleware/ # Auth middleware
│ │ ├── models/ # Mongoose models
│ │ ├── routes/ # API routes
│ │ ├── scraper/ # Playwright scraper engine
│ │ └── utils/ # Encryption, logging
│ └── package.json
├── frontend/
│ ├── src/
│ │ ├── components/ # React components
│ │ ├── pages/ # Page components
│ │ └── stores/ # Zustand stores
│ └── package.json
├── Dockerfile
├── docker-compose.yml
└── package.json
```
## Security Considerations
1. **Use strong passwords** - The master password is your only line of defense
2. **Secure your secrets** - Never commit `.env` files
3. **Use HTTPS** - Always deploy behind HTTPS in production
4. **Rate limiting** - Login attempts are rate-limited (5 per 15 minutes)
5. **Session encryption** - All stored cookies are AES-256-GCM encrypted
6. **MongoDB Security** - Use authentication and TLS for your MongoDB connection
## License
Private - All rights reserved.

3114
backend/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

40
backend/package.json Normal file
View File

@@ -0,0 +1,40 @@
{
"name": "osint-backend",
"version": "1.0.0",
"description": "OSINT Platform Backend",
"type": "module",
"main": "dist/index.js",
"scripts": {
"dev": "tsx watch src/index.ts",
"build": "tsc",
"start": "node dist/index.js"
},
"dependencies": {
"express": "^4.18.2",
"cors": "^2.8.5",
"helmet": "^7.1.0",
"bcryptjs": "^2.4.3",
"jsonwebtoken": "^9.0.2",
"cookie-parser": "^1.4.6",
"mongoose": "^8.2.1",
"playwright": "^1.42.1",
"playwright-extra": "^4.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"dotenv": "^16.4.5",
"uuid": "^9.0.1",
"winston": "^3.11.0",
"express-rate-limit": "^7.2.0",
"socket.io": "^4.7.5"
},
"devDependencies": {
"@types/express": "^4.17.21",
"@types/cors": "^2.8.17",
"@types/bcryptjs": "^2.4.6",
"@types/jsonwebtoken": "^9.0.6",
"@types/cookie-parser": "^1.4.7",
"@types/uuid": "^9.0.8",
"@types/node": "^20.11.24",
"typescript": "^5.4.2",
"tsx": "^4.7.1"
}
}

View File

@@ -0,0 +1,35 @@
import mongoose from 'mongoose';
import { logger } from '../utils/logger.js';
export async function connectDatabase(): Promise<void> {
const mongoUri = process.env.MONGODB_URI;
if (!mongoUri) {
throw new Error('MONGODB_URI environment variable is required');
}
try {
await mongoose.connect(mongoUri, {
dbName: 'osint_platform',
});
logger.info('Connected to MongoDB');
mongoose.connection.on('error', (err) => {
logger.error('MongoDB connection error:', err);
});
mongoose.connection.on('disconnected', () => {
logger.warn('MongoDB disconnected');
});
} catch (error) {
logger.error('Failed to connect to MongoDB:', error);
throw error;
}
}
export async function closeDatabase(): Promise<void> {
await mongoose.connection.close();
logger.info('MongoDB connection closed');
}

131
backend/src/index.ts Normal file
View File

@@ -0,0 +1,131 @@
import express from 'express';
import cors from 'cors';
import helmet from 'helmet';
import cookieParser from 'cookie-parser';
import { createServer } from 'http';
import { Server } from 'socket.io';
import dotenv from 'dotenv';
import path from 'path';
import { fileURLToPath } from 'url';
import { authRouter } from './routes/auth.js';
import { targetsRouter } from './routes/targets.js';
import { sessionsRouter } from './routes/sessions.js';
import { scraperRouter } from './routes/scraper.js';
import { authMiddleware } from './middleware/auth.js';
import { connectDatabase } from './database/index.js';
import { logger } from './utils/logger.js';
import { ScraperManager } from './scraper/manager.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
dotenv.config({ path: path.join(__dirname, '../../.env') });
const app = express();
const httpServer = createServer(app);
const io = new Server(httpServer, {
cors: {
origin: process.env.FRONTEND_URL || 'http://localhost:5173',
credentials: true,
},
});
const PORT = process.env.PORT || 3001;
// Initialize scraper manager with socket.io
const scraperManager = new ScraperManager(io);
app.set('scraperManager', scraperManager);
app.set('io', io);
// Middleware
app.use(helmet({
contentSecurityPolicy: false,
}));
app.use(cors({
origin: process.env.FRONTEND_URL || 'http://localhost:5173',
credentials: true,
}));
app.use(express.json());
app.use(cookieParser());
// Serve static files from frontend build in production
if (process.env.NODE_ENV === 'production') {
app.use(express.static(path.join(__dirname, '../../frontend/dist')));
}
// API Routes
app.use('/api/auth', authRouter);
app.use('/api/targets', authMiddleware, targetsRouter);
app.use('/api/sessions', authMiddleware, sessionsRouter);
app.use('/api/scraper', authMiddleware, scraperRouter);
// Health check
app.get('/api/health', (req, res) => {
res.json({ status: 'ok', timestamp: new Date().toISOString() });
});
// Catch-all for SPA routing in production
if (process.env.NODE_ENV === 'production') {
app.get('*', (req, res) => {
res.sendFile(path.join(__dirname, '../../frontend/dist/index.html'));
});
}
// Socket.io authentication
io.use((socket, next) => {
const token = socket.handshake.auth.token;
if (!token) {
return next(new Error('Authentication required'));
}
// Token validation would go here
next();
});
io.on('connection', (socket) => {
logger.info(`Client connected: ${socket.id}`);
socket.on('subscribe:scraper', (scraperId: string) => {
socket.join(`scraper:${scraperId}`);
});
socket.on('unsubscribe:scraper', (scraperId: string) => {
socket.leave(`scraper:${scraperId}`);
});
socket.on('disconnect', () => {
logger.info(`Client disconnected: ${socket.id}`);
});
});
// Global error handlers
process.on('unhandledRejection', (reason, promise) => {
logger.error('Unhandled Rejection at:', promise, 'reason:', reason);
});
process.on('uncaughtException', (error) => {
logger.error('Uncaught Exception:', error);
// Important: give the logger time to write before exiting
setTimeout(() => {
process.exit(1);
}, 1000);
});
// Start server with database connection
async function start() {
try {
await connectDatabase();
httpServer.listen(PORT, () => {
logger.info(`🔒 OSINT Platform Backend running on port ${PORT}`);
logger.info(`📊 Environment: ${process.env.NODE_ENV || 'development'}`);
});
} catch (error) {
logger.error('Failed to start server:', error);
process.exit(1);
}
}
start();
export { app, io };

View File

@@ -0,0 +1,55 @@
import { Request, Response, NextFunction } from 'express';
import jwt from 'jsonwebtoken';
import { logger } from '../utils/logger.js';
export interface AuthRequest extends Request {
userId?: string;
}
export function authMiddleware(req: AuthRequest, res: Response, next: NextFunction): void {
try {
// Check for token in Authorization header or cookie
const authHeader = req.headers.authorization;
const cookieToken = req.cookies?.auth_token;
let token: string | undefined;
if (authHeader && authHeader.startsWith('Bearer ')) {
token = authHeader.substring(7);
} else if (cookieToken) {
token = cookieToken;
}
if (!token) {
res.status(401).json({ error: 'Authentication required' });
return;
}
const jwtSecret = process.env.JWT_SECRET;
if (!jwtSecret) {
logger.error('JWT_SECRET not configured');
res.status(500).json({ error: 'Server configuration error' });
return;
}
const decoded = jwt.verify(token, jwtSecret) as { authenticated: boolean };
if (!decoded.authenticated) {
res.status(401).json({ error: 'Invalid token' });
return;
}
next();
} catch (error) {
if (error instanceof jwt.TokenExpiredError) {
res.status(401).json({ error: 'Token expired' });
return;
}
if (error instanceof jwt.JsonWebTokenError) {
res.status(401).json({ error: 'Invalid token' });
return;
}
logger.error('Auth middleware error:', error);
res.status(500).json({ error: 'Authentication error' });
}
}

View File

@@ -0,0 +1,52 @@
import mongoose, { Document, Schema } from 'mongoose';
export interface IScraperLog {
level: string;
message: string;
timestamp: Date;
}
export interface IScraperJob extends Document {
targetId?: mongoose.Types.ObjectId;
profileId?: mongoose.Types.ObjectId;
platform: string;
status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
progress: number;
result?: Record<string, any>;
error?: string;
logs: IScraperLog[];
startedAt?: Date;
completedAt?: Date;
createdAt: Date;
}
const ScraperLogSchema = new Schema<IScraperLog>({
level: { type: String, required: true },
message: { type: String, required: true },
timestamp: { type: Date, default: Date.now },
}, { _id: false });
const ScraperJobSchema = new Schema<IScraperJob>({
targetId: { type: Schema.Types.ObjectId, ref: 'Target' },
profileId: { type: Schema.Types.ObjectId },
platform: { type: String, required: true },
status: {
type: String,
enum: ['pending', 'running', 'completed', 'failed', 'cancelled'],
default: 'pending'
},
progress: { type: Number, default: 0 },
result: { type: Schema.Types.Mixed },
error: { type: String },
logs: [ScraperLogSchema],
startedAt: { type: Date },
completedAt: { type: Date },
}, {
timestamps: true,
});
// Index for efficient querying
ScraperJobSchema.index({ status: 1, createdAt: -1 });
ScraperJobSchema.index({ targetId: 1 });
export const ScraperJob = mongoose.model<IScraperJob>('ScraperJob', ScraperJobSchema);

View File

@@ -0,0 +1,33 @@
import mongoose, { Document, Schema } from 'mongoose';
export interface ISession extends Document {
platform: string;
sessionName: string;
cookiesEncrypted: string;
localStorageEncrypted?: string;
userAgent?: string;
proxy?: string;
status: 'active' | 'expired' | 'invalid';
lastValidated?: Date;
createdAt: Date;
updatedAt: Date;
}
const SessionSchema = new Schema<ISession>({
platform: { type: String, required: true, unique: true },
sessionName: { type: String, required: true },
cookiesEncrypted: { type: String, required: true },
localStorageEncrypted: { type: String },
userAgent: { type: String },
proxy: { type: String },
status: {
type: String,
enum: ['active', 'expired', 'invalid'],
default: 'active'
},
lastValidated: { type: Date },
}, {
timestamps: true,
});
export const Session = mongoose.model<ISession>('Session', SessionSchema);

View File

@@ -0,0 +1,47 @@
import mongoose, { Document, Schema } from 'mongoose';
export interface ISocialProfile {
_id: mongoose.Types.ObjectId;
platform: string;
username?: string;
profileUrl?: string;
profileData?: Record<string, any>;
lastScraped?: Date;
createdAt: Date;
}
export interface ITarget extends Document {
name: string;
notes?: string;
profiles: ISocialProfile[];
createdAt: Date;
updatedAt: Date;
}
const SocialProfileSchema = new Schema<ISocialProfile>({
platform: { type: String, required: true },
username: { type: String },
profileUrl: { type: String },
profileData: { type: Schema.Types.Mixed },
lastScraped: { type: Date },
createdAt: { type: Date, default: Date.now },
});
const TargetSchema = new Schema<ITarget>({
name: { type: String, required: true },
notes: { type: String },
profiles: [SocialProfileSchema],
}, {
timestamps: true,
});
// Virtual for profile count
TargetSchema.virtual('profileCount').get(function() {
return this.profiles?.length || 0;
});
// Ensure virtuals are serialized
TargetSchema.set('toJSON', { virtuals: true });
TargetSchema.set('toObject', { virtuals: true });
export const Target = mongoose.model<ITarget>('Target', TargetSchema);

View File

@@ -0,0 +1,3 @@
export { Target, type ITarget, type ISocialProfile } from './Target.js';
export { Session, type ISession } from './Session.js';
export { ScraperJob, type IScraperJob, type IScraperLog } from './ScraperJob.js';

113
backend/src/routes/auth.ts Normal file
View File

@@ -0,0 +1,113 @@
import { Router, Request, Response } from 'express';
import bcrypt from 'bcryptjs';
import jwt from 'jsonwebtoken';
import rateLimit from 'express-rate-limit';
import { logger } from '../utils/logger.js';
export const authRouter = Router();
// Rate limiting for auth endpoints
const authLimiter = rateLimit({
windowMs: 15 * 60 * 1000, // 15 minutes
max: 5, // 5 attempts per window
message: { error: 'Too many authentication attempts, please try again later' },
standardHeaders: true,
legacyHeaders: false,
});
// Login with master password
authRouter.post('/login', authLimiter, async (req: Request, res: Response) => {
try {
const { password } = req.body;
if (!password) {
res.status(400).json({ error: 'Password is required' });
return;
}
const masterPassword = process.env.MASTER_PASSWORD;
if (!masterPassword) {
logger.error('MASTER_PASSWORD not configured');
res.status(500).json({ error: 'Server configuration error' });
return;
}
// Simple comparison for now - in production you'd hash the stored password
const isValid = password === masterPassword;
if (!isValid) {
logger.warn('Failed login attempt');
res.status(401).json({ error: 'Invalid password' });
return;
}
const jwtSecret = process.env.JWT_SECRET;
if (!jwtSecret) {
logger.error('JWT_SECRET not configured');
res.status(500).json({ error: 'Server configuration error' });
return;
}
const token = jwt.sign(
{ authenticated: true },
jwtSecret as string,
{ expiresIn: (process.env.SESSION_EXPIRY || '24h') as any }
);
// Set HTTP-only cookie
res.cookie('auth_token', token, {
httpOnly: true,
secure: process.env.NODE_ENV === 'production',
sameSite: 'strict',
maxAge: 24 * 60 * 60 * 1000, // 24 hours
});
logger.info('Successful login');
res.json({
success: true,
token,
expiresIn: process.env.SESSION_EXPIRY || '24h'
});
} catch (error) {
logger.error('Login error:', error);
res.status(500).json({ error: 'Authentication failed' });
}
});
// Verify token
authRouter.get('/verify', (req: Request, res: Response) => {
try {
const authHeader = req.headers.authorization;
const cookieToken = req.cookies?.auth_token;
let token: string | undefined;
if (authHeader && authHeader.startsWith('Bearer ')) {
token = authHeader.substring(7);
} else if (cookieToken) {
token = cookieToken;
}
if (!token) {
res.status(401).json({ authenticated: false });
return;
}
const jwtSecret = process.env.JWT_SECRET;
if (!jwtSecret) {
res.status(500).json({ error: 'Server configuration error' });
return;
}
jwt.verify(token, jwtSecret);
res.json({ authenticated: true });
} catch (error) {
res.status(401).json({ authenticated: false });
}
});
// Logout
authRouter.post('/logout', (req: Request, res: Response) => {
res.clearCookie('auth_token');
res.json({ success: true });
});

View File

@@ -0,0 +1,199 @@
import { Router, Request, Response } from 'express';
import mongoose from 'mongoose';
import { ScraperJob } from '../models/ScraperJob.js';
import { Target } from '../models/Target.js';
import { logger } from '../utils/logger.js';
import { ScraperManager } from '../scraper/manager.js';
export const scraperRouter = Router();
// Get all jobs
scraperRouter.get('/jobs', async (req: Request, res: Response) => {
try {
const limit = parseInt(req.query.limit as string) || 50;
const jobs = await ScraperJob.find()
.sort({ createdAt: -1 })
.limit(limit)
.lean();
// Populate target names
const targetIds = [...new Set(jobs.filter(j => j.targetId).map(j => j.targetId!.toString()))];
const targets = await Target.find({ _id: { $in: targetIds } }).select('name').lean();
const targetMap = new Map(targets.map(t => [t._id.toString(), t.name]));
const formattedJobs = jobs.map(job => ({
id: job._id,
target_id: job.targetId,
profile_id: job.profileId,
platform: job.platform,
status: job.status,
progress: job.progress,
result: job.result ? JSON.stringify(job.result) : null,
error: job.error,
target_name: job.targetId ? targetMap.get(job.targetId.toString()) : null,
started_at: job.startedAt,
completed_at: job.completedAt,
created_at: job.createdAt,
}));
res.json(formattedJobs);
} catch (error) {
logger.error('Error fetching jobs:', error);
res.status(500).json({ error: 'Failed to fetch jobs' });
}
});
// Get job by ID with logs
scraperRouter.get('/jobs/:id', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const job = await ScraperJob.findById(id).lean();
if (!job) {
res.status(404).json({ error: 'Job not found' });
return;
}
// Get target name
let targetName = null;
if (job.targetId) {
const target = await Target.findById(job.targetId).select('name').lean();
targetName = target?.name;
}
res.json({
id: job._id,
target_id: job.targetId,
profile_id: job.profileId,
platform: job.platform,
status: job.status,
progress: job.progress,
result: job.result,
error: job.error,
target_name: targetName,
started_at: job.startedAt,
completed_at: job.completedAt,
created_at: job.createdAt,
logs: job.logs,
});
} catch (error) {
logger.error('Error fetching job:', error);
res.status(500).json({ error: 'Failed to fetch job' });
}
});
// Start a new scrape job
scraperRouter.post('/start', async (req: Request, res: Response) => {
try {
const { target_id, profile_id, platform, profile_url } = req.body;
if (!platform) {
res.status(400).json({ error: 'Platform is required' });
return;
}
// Create job record
const job = new ScraperJob({
targetId: target_id ? new mongoose.Types.ObjectId(target_id) : undefined,
profileId: profile_id ? new mongoose.Types.ObjectId(profile_id) : undefined,
platform,
status: 'pending',
progress: 0,
logs: [],
});
await job.save();
// Get scraper manager and start job
const scraperManager = req.app.get('scraperManager') as ScraperManager;
scraperManager.startJob({
jobId: job._id.toString(),
platform,
profileUrl: profile_url,
targetId: target_id,
profileId: profile_id,
});
logger.info(`Started scraper job: ${job._id} for ${platform}`);
res.status(201).json({
id: job._id,
platform: job.platform,
status: job.status,
progress: job.progress,
created_at: job.createdAt,
});
} catch (error) {
logger.error('Error starting job:', error);
res.status(500).json({ error: 'Failed to start job' });
}
});
// Cancel a job
scraperRouter.post('/jobs/:id/cancel', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const scraperManager = req.app.get('scraperManager') as ScraperManager;
await scraperManager.cancelJob(id);
const job = await ScraperJob.findById(id).lean();
res.json({
id: job?._id,
status: job?.status,
});
} catch (error) {
logger.error('Error cancelling job:', error);
res.status(500).json({ error: 'Failed to cancel job' });
}
});
// Get active jobs summary
scraperRouter.get('/status', async (req: Request, res: Response) => {
try {
const [pending, running, completed, failed] = await Promise.all([
ScraperJob.countDocuments({ status: 'pending' }),
ScraperJob.countDocuments({ status: 'running' }),
ScraperJob.countDocuments({ status: 'completed' }),
ScraperJob.countDocuments({ status: 'failed' }),
]);
const activeJobs = await ScraperJob.find({ status: { $in: ['pending', 'running'] } })
.sort({ createdAt: -1 })
.select('_id platform status progress startedAt')
.lean();
res.json({
counts: { pending, running, completed, failed },
activeJobs: activeJobs.map(j => ({
id: j._id,
platform: j.platform,
status: j.status,
progress: j.progress,
started_at: j.startedAt,
})),
});
} catch (error) {
logger.error('Error fetching scraper status:', error);
res.status(500).json({ error: 'Failed to fetch status' });
}
});
// Get logs for a job
scraperRouter.get('/jobs/:id/logs', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const job = await ScraperJob.findById(id).select('logs').lean();
if (!job) {
res.status(404).json({ error: 'Job not found' });
return;
}
res.json(job.logs || []);
} catch (error) {
logger.error('Error fetching logs:', error);
res.status(500).json({ error: 'Failed to fetch logs' });
}
});

View File

@@ -0,0 +1,225 @@
import { Router, Request, Response } from 'express';
import { Session } from '../models/Session.js';
import { encrypt, decrypt } from '../utils/encryption.js';
import { logger } from '../utils/logger.js';
export const sessionsRouter = Router();
// Get all sessions (without sensitive data)
sessionsRouter.get('/', async (req: Request, res: Response) => {
try {
const sessions = await Session.find()
.select('-cookiesEncrypted -localStorageEncrypted')
.sort({ updatedAt: -1 })
.lean();
const formattedSessions = sessions.map(s => ({
id: s._id,
platform: s.platform,
session_name: s.sessionName,
user_agent: s.userAgent,
proxy: s.proxy,
status: s.status,
last_validated: s.lastValidated,
created_at: s.createdAt,
updated_at: s.updatedAt,
}));
res.json(formattedSessions);
} catch (error) {
logger.error('Error fetching sessions:', error);
res.status(500).json({ error: 'Failed to fetch sessions' });
}
});
// Get session by platform
sessionsRouter.get('/platform/:platform', async (req: Request, res: Response) => {
try {
const { platform } = req.params;
const session = await Session.findOne({ platform })
.select('-cookiesEncrypted -localStorageEncrypted')
.lean();
if (!session) {
res.status(404).json({ error: 'Session not found for platform' });
return;
}
res.json({
id: session._id,
platform: session.platform,
session_name: session.sessionName,
user_agent: session.userAgent,
proxy: session.proxy,
status: session.status,
last_validated: session.lastValidated,
created_at: session.createdAt,
updated_at: session.updatedAt,
});
} catch (error) {
logger.error('Error fetching session:', error);
res.status(500).json({ error: 'Failed to fetch session' });
}
});
// Create or update session
sessionsRouter.post('/', async (req: Request, res: Response) => {
try {
const { platform, session_name, cookies, local_storage, user_agent, proxy } = req.body;
if (!platform || !session_name || !cookies) {
res.status(400).json({ error: 'Platform, session_name, and cookies are required' });
return;
}
// Encrypt sensitive data
const cookiesEncrypted = encrypt(JSON.stringify(cookies));
const localStorageEncrypted = local_storage ? encrypt(JSON.stringify(local_storage)) : undefined;
// Upsert session
const session = await Session.findOneAndUpdate(
{ platform },
{
platform,
sessionName: session_name,
cookiesEncrypted,
localStorageEncrypted,
userAgent: user_agent,
proxy,
status: 'active',
},
{ upsert: true, new: true }
).lean();
logger.info(`Saved session for ${platform}`);
res.status(201).json({
id: session._id,
platform: session.platform,
session_name: session.sessionName,
user_agent: session.userAgent,
proxy: session.proxy,
status: session.status,
created_at: session.createdAt,
updated_at: session.updatedAt,
});
} catch (error) {
logger.error('Error saving session:', error);
res.status(500).json({ error: 'Failed to save session' });
}
});
// Get decrypted session data (for internal scraper use)
sessionsRouter.get('/decrypt/:platform', async (req: Request, res: Response) => {
try {
const { platform } = req.params;
const session = await Session.findOne({ platform }).lean();
if (!session) {
res.status(404).json({ error: 'Session not found' });
return;
}
const cookies = JSON.parse(decrypt(session.cookiesEncrypted));
const localStorage = session.localStorageEncrypted
? JSON.parse(decrypt(session.localStorageEncrypted))
: null;
res.json({
id: session._id,
platform: session.platform,
session_name: session.sessionName,
cookies,
localStorage,
user_agent: session.userAgent,
proxy: session.proxy,
status: session.status,
});
} catch (error) {
logger.error('Error decrypting session:', error);
res.status(500).json({ error: 'Failed to decrypt session' });
}
});
// Validate session
sessionsRouter.post('/validate/:platform', async (req: Request, res: Response) => {
try {
const { platform } = req.params;
const session = await Session.findOneAndUpdate(
{ platform },
{ lastValidated: new Date() },
{ new: true }
).select('-cookiesEncrypted -localStorageEncrypted').lean();
if (!session) {
res.status(404).json({ error: 'Session not found' });
return;
}
res.json({
id: session._id,
platform: session.platform,
session_name: session.sessionName,
status: session.status,
last_validated: session.lastValidated,
});
} catch (error) {
logger.error('Error validating session:', error);
res.status(500).json({ error: 'Failed to validate session' });
}
});
// Update session status
sessionsRouter.patch('/:id/status', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const { status } = req.body;
if (!status || !['active', 'expired', 'invalid'].includes(status)) {
res.status(400).json({ error: 'Valid status required (active, expired, invalid)' });
return;
}
const session = await Session.findByIdAndUpdate(
id,
{ status },
{ new: true }
).select('-cookiesEncrypted -localStorageEncrypted').lean();
if (!session) {
res.status(404).json({ error: 'Session not found' });
return;
}
res.json({
id: session._id,
platform: session.platform,
session_name: session.sessionName,
status: session.status,
updated_at: session.updatedAt,
});
} catch (error) {
logger.error('Error updating session status:', error);
res.status(500).json({ error: 'Failed to update session status' });
}
});
// Delete session
sessionsRouter.delete('/:id', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const result = await Session.findByIdAndDelete(id);
if (!result) {
res.status(404).json({ error: 'Session not found' });
return;
}
logger.info(`Deleted session: ${id}`);
res.json({ success: true });
} catch (error) {
logger.error('Error deleting session:', error);
res.status(500).json({ error: 'Failed to delete session' });
}
});

View File

@@ -0,0 +1,199 @@
import { Router, Request, Response } from 'express';
import { Target } from '../models/Target.js';
import { logger } from '../utils/logger.js';
export const targetsRouter = Router();
// Get all targets
targetsRouter.get('/', async (req: Request, res: Response) => {
try {
const targets = await Target.find()
.sort({ updatedAt: -1 })
.lean();
// Add profile count
const targetsWithCount = targets.map(t => ({
...t,
id: t._id,
profile_count: t.profiles?.length || 0,
}));
res.json(targetsWithCount);
} catch (error) {
logger.error('Error fetching targets:', error);
res.status(500).json({ error: 'Failed to fetch targets' });
}
});
// Get single target with profiles
targetsRouter.get('/:id', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const target = await Target.findById(id).lean();
if (!target) {
res.status(404).json({ error: 'Target not found' });
return;
}
res.json({
...target,
id: target._id,
profiles: target.profiles?.map(p => ({
...p,
id: p._id,
target_id: target._id,
profile_url: p.profileUrl,
profile_data: p.profileData ? JSON.stringify(p.profileData) : null,
last_scraped: p.lastScraped,
created_at: p.createdAt,
})) || [],
});
} catch (error) {
logger.error('Error fetching target:', error);
res.status(500).json({ error: 'Failed to fetch target' });
}
});
// Create target
targetsRouter.post('/', async (req: Request, res: Response) => {
try {
const { name, notes } = req.body;
if (!name) {
res.status(400).json({ error: 'Name is required' });
return;
}
const target = new Target({ name, notes, profiles: [] });
await target.save();
logger.info(`Created target: ${name} (${target._id})`);
res.status(201).json({
...target.toObject(),
id: target._id,
profile_count: 0,
});
} catch (error) {
logger.error('Error creating target:', error);
res.status(500).json({ error: 'Failed to create target' });
}
});
// Update target
targetsRouter.put('/:id', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const { name, notes } = req.body;
const target = await Target.findByIdAndUpdate(
id,
{ name, notes },
{ new: true }
).lean();
if (!target) {
res.status(404).json({ error: 'Target not found' });
return;
}
res.json({ ...target, id: target._id });
} catch (error) {
logger.error('Error updating target:', error);
res.status(500).json({ error: 'Failed to update target' });
}
});
// Delete target
targetsRouter.delete('/:id', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const result = await Target.findByIdAndDelete(id);
if (!result) {
res.status(404).json({ error: 'Target not found' });
return;
}
logger.info(`Deleted target: ${id}`);
res.json({ success: true });
} catch (error) {
logger.error('Error deleting target:', error);
res.status(500).json({ error: 'Failed to delete target' });
}
});
// Add profile to target
targetsRouter.post('/:id/profiles', async (req: Request, res: Response) => {
try {
const { id } = req.params;
const { platform, username, profile_url } = req.body;
if (!platform) {
res.status(400).json({ error: 'Platform is required' });
return;
}
const target = await Target.findById(id);
if (!target) {
res.status(404).json({ error: 'Target not found' });
return;
}
const newProfile = {
platform,
username: username || undefined,
profileUrl: profile_url || undefined,
createdAt: new Date(),
};
target.profiles.push(newProfile as any);
await target.save();
const addedProfile = target.profiles[target.profiles.length - 1];
logger.info(`Added ${platform} profile to target ${id}`);
res.status(201).json({
id: addedProfile._id,
target_id: id,
platform: addedProfile.platform,
username: addedProfile.username,
profile_url: addedProfile.profileUrl,
created_at: addedProfile.createdAt,
});
} catch (error) {
logger.error('Error adding profile:', error);
res.status(500).json({ error: 'Failed to add profile' });
}
});
// Delete profile
targetsRouter.delete('/:id/profiles/:profileId', async (req: Request, res: Response) => {
try {
const { id, profileId } = req.params;
const target = await Target.findById(id);
if (!target) {
res.status(404).json({ error: 'Target not found' });
return;
}
const profileIndex = target.profiles.findIndex(
p => p._id.toString() === profileId
);
if (profileIndex === -1) {
res.status(404).json({ error: 'Profile not found' });
return;
}
target.profiles.splice(profileIndex, 1);
await target.save();
logger.info(`Deleted profile ${profileId} from target ${id}`);
res.json({ success: true });
} catch (error) {
logger.error('Error deleting profile:', error);
res.status(500).json({ error: 'Failed to delete profile' });
}
});

View File

@@ -0,0 +1,509 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { Server } from 'socket.io';
import { chromium, Browser, BrowserContext, Page } from 'playwright';
import mongoose from 'mongoose';
import { ScraperJob } from '../models/ScraperJob.js';
import { Session } from '../models/Session.js';
import { Target } from '../models/Target.js';
import { decrypt } from '../utils/encryption.js';
import { logger } from '../utils/logger.js';
declare const document: any;
declare const window: any;
export interface ScraperJobConfig {
jobId: string;
platform: string;
profileUrl?: string;
targetId?: string;
profileId?: string;
}
interface ActiveJob {
config: ScraperJobConfig;
browser?: Browser;
context?: BrowserContext;
page?: Page;
abortController: AbortController;
}
export class ScraperManager {
private io: Server;
private activeJobs: Map<string, ActiveJob> = new Map();
private browser: Browser | null = null;
constructor(io: Server) {
this.io = io;
}
private async log(jobId: string, level: string, message: string): Promise<void> {
await ScraperJob.findByIdAndUpdate(jobId, {
$push: {
logs: {
level,
message,
timestamp: new Date(),
}
}
});
// Emit to socket
this.io.to(`scraper:${jobId}`).emit('scraper:log', {
jobId,
level,
message,
timestamp: new Date().toISOString(),
});
logger[level as 'info' | 'warn' | 'error'](`[Job ${jobId}] ${message}`);
}
private async updateJobStatus(
jobId: string,
status: string,
progress?: number,
result?: any,
error?: string
): Promise<void> {
const update: any = { status };
if (progress !== undefined) update.progress = progress;
if (result !== undefined) update.result = result;
if (error !== undefined) update.error = error;
if (status === 'running') update.startedAt = new Date();
if (status === 'completed' || status === 'failed' || status === 'cancelled') {
update.completedAt = new Date();
}
await ScraperJob.findByIdAndUpdate(jobId, update);
// Emit status update
this.io.to(`scraper:${jobId}`).emit('scraper:status', {
jobId,
status,
progress,
result: result ? result : undefined,
error,
});
// Also emit to general channel
this.io.emit('scraper:jobUpdate', { jobId, status, progress });
}
async startJob(config: ScraperJobConfig): Promise<void> {
const abortController = new AbortController();
const activeJob: ActiveJob = {
config,
abortController,
};
this.activeJobs.set(config.jobId, activeJob);
// Run asynchronously
this.runJob(config, abortController.signal).catch((error) => {
logger.error(`Job ${config.jobId} failed:`, error);
});
}
private async runJob(config: ScraperJobConfig, signal: AbortSignal): Promise<void> {
const { jobId, platform, profileUrl } = config;
try {
await this.updateJobStatus(jobId, 'running', 0);
await this.log(jobId, 'info', `Starting scrape for platform: ${platform}`);
// Load session from vault
const session = await this.loadSession(platform);
if (!session) {
throw new Error(`No session found for platform: ${platform}`);
}
await this.log(jobId, 'info', `Loaded session: ${session.sessionName}`);
await this.updateJobStatus(jobId, 'running', 10);
// Initialize browser
await this.log(jobId, 'info', 'Initializing browser...');
const browser = await this.getBrowser();
if (signal.aborted) {
throw new Error('Job cancelled');
}
// Create context with session
const context = await browser.newContext({
userAgent: session.userAgent || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
viewport: { width: 1920, height: 1080 },
locale: 'en-US',
});
// Add cookies
if (session.cookies && Array.isArray(session.cookies)) {
await context.addCookies(session.cookies);
await this.log(jobId, 'info', `Loaded ${session.cookies.length} cookies`);
}
await this.updateJobStatus(jobId, 'running', 20);
const page = await context.newPage();
// Store references
const activeJob = this.activeJobs.get(jobId);
if (activeJob) {
activeJob.context = context;
activeJob.page = page;
}
if (signal.aborted) {
await context.close();
throw new Error('Job cancelled');
}
// Run platform-specific scraper
let result: any;
switch (platform.toLowerCase()) {
case 'x':
case 'twitter':
result = await this.scrapeTwitter(jobId, page, profileUrl, signal);
break;
case 'instagram':
result = await this.scrapeInstagram(jobId, page, profileUrl, signal);
break;
case 'linkedin':
result = await this.scrapeLinkedIn(jobId, page, profileUrl, signal);
break;
case 'facebook':
result = await this.scrapeFacebook(jobId, page, profileUrl, signal);
break;
default:
result = await this.scrapeGeneric(jobId, page, profileUrl || '', signal);
}
// Close context
await context.close();
// Save result to profile if profile_id provided
if (config.profileId && config.targetId && result) {
await Target.updateOne(
{ _id: new mongoose.Types.ObjectId(config.targetId), 'profiles._id': new mongoose.Types.ObjectId(config.profileId) },
{
$set: {
'profiles.$.profileData': result,
'profiles.$.lastScraped': new Date()
}
}
);
}
await this.updateJobStatus(jobId, 'completed', 100, result);
await this.log(jobId, 'info', 'Scrape completed successfully');
} catch (error: any) {
const errorMessage = error.message || 'Unknown error';
await this.log(jobId, 'error', `Scrape failed: ${errorMessage}`);
await this.updateJobStatus(jobId, 'failed', undefined, undefined, errorMessage);
} finally {
this.activeJobs.delete(jobId);
}
}
private async loadSession(platform: string): Promise<any> {
const session = await Session.findOne({ platform, status: 'active' }).lean();
if (!session) {
return null;
}
return {
...session,
cookies: JSON.parse(decrypt(session.cookiesEncrypted)),
localStorage: session.localStorageEncrypted
? JSON.parse(decrypt(session.localStorageEncrypted))
: null,
};
}
private async getBrowser(): Promise<Browser> {
if (!this.browser || !this.browser.isConnected()) {
this.browser = await chromium.launch({
headless: true,
args: [
'--disable-blink-features=AutomationControlled',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-web-security',
'--disable-features=IsolateOrigins,site-per-process',
],
});
}
return this.browser;
}
// Platform-specific scrapers
private async scrapeTwitter(
jobId: string,
page: Page,
profileUrl: string | undefined,
signal: AbortSignal
): Promise<any> {
const url = profileUrl || 'https://x.com/home';
await this.log(jobId, 'info', `Navigating to: ${url}`);
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
await this.updateJobStatus(jobId, 'running', 40);
if (signal.aborted) throw new Error('Job cancelled');
// Wait for content to load
await page.waitForTimeout(2000);
await this.log(jobId, 'info', 'Extracting profile data...');
await this.updateJobStatus(jobId, 'running', 60);
// Extract profile data
const profileData = await page.evaluate(() => {
const doc = document as any;
const win = window as any;
const data: any = {
url: win.location.href,
scraped_at: new Date().toISOString(),
};
// Try to extract profile info
const nameElement = doc.querySelector('[data-testid="UserName"]');
if (nameElement) {
data.display_name = nameElement.querySelector('span')?.textContent;
data.username = nameElement.querySelectorAll('span')[1]?.textContent;
}
const bioElement = doc.querySelector('[data-testid="UserDescription"]');
if (bioElement) {
data.bio = bioElement.textContent;
}
// Extract stats
const statsElements = doc.querySelectorAll('[href*="/following"], [href*="/followers"]');
statsElements.forEach((el: any) => {
const href = el.getAttribute('href');
const text = el.textContent;
if (href?.includes('following')) {
data.following = text;
} else if (href?.includes('followers')) {
data.followers = text;
}
});
// Get recent tweets
const tweets: any[] = [];
doc.querySelectorAll('[data-testid="tweet"]').forEach((tweet: any, i: number) => {
if (i < 10) { // Limit to 10 tweets
tweets.push({
text: tweet.querySelector('[data-testid="tweetText"]')?.textContent,
timestamp: tweet.querySelector('time')?.getAttribute('datetime'),
});
}
});
data.recent_tweets = tweets;
return data;
});
await this.updateJobStatus(jobId, 'running', 80);
await this.log(jobId, 'info', `Extracted profile: ${profileData.username || 'unknown'}`);
return profileData;
}
private async scrapeInstagram(
jobId: string,
page: Page,
profileUrl: string | undefined,
signal: AbortSignal
): Promise<any> {
const url = profileUrl || 'https://instagram.com';
await this.log(jobId, 'info', `Navigating to: ${url}`);
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
await this.updateJobStatus(jobId, 'running', 40);
if (signal.aborted) throw new Error('Job cancelled');
await page.waitForTimeout(2000);
await this.log(jobId, 'info', 'Extracting profile data...');
await this.updateJobStatus(jobId, 'running', 60);
const profileData = await page.evaluate(() => {
const doc = document as any;
const win = window as any;
const data: any = {
url: win.location.href,
scraped_at: new Date().toISOString(),
};
// Extract from meta tags and visible elements
const ogTitle = doc.querySelector('meta[property="og:title"]');
if (ogTitle) {
data.title = ogTitle.getAttribute('content');
}
const ogDescription = document.querySelector('meta[property="og:description"]');
if (ogDescription) {
data.description = ogDescription.getAttribute('content');
}
// Try to parse stats from description
const statsMatch = data.description?.match(/(\d+(?:,\d+)*(?:\.\d+)?[KMB]?)\s+Followers/i);
if (statsMatch) {
data.followers = statsMatch[1];
}
// Get profile picture
const profilePic = doc.querySelector('img[alt*="profile picture"]');
if (profilePic) {
data.profile_picture = profilePic.getAttribute('src');
}
return data;
});
await this.updateJobStatus(jobId, 'running', 80);
return profileData;
}
private async scrapeLinkedIn(
jobId: string,
page: Page,
profileUrl: string | undefined,
signal: AbortSignal
): Promise<any> {
const url = profileUrl || 'https://linkedin.com';
await this.log(jobId, 'info', `Navigating to: ${url}`);
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
await this.updateJobStatus(jobId, 'running', 40);
if (signal.aborted) throw new Error('Job cancelled');
await page.waitForTimeout(2000);
await this.updateJobStatus(jobId, 'running', 60);
const profileData = await page.evaluate(() => {
const doc = document as any;
const win = window as any;
const data: any = {
url: win.location.href,
scraped_at: new Date().toISOString(),
};
// Extract profile info
const nameElement = doc.querySelector('h1');
if (nameElement) {
data.name = nameElement.textContent?.trim();
}
const headlineElement = document.querySelector('.text-body-medium');
if (headlineElement) {
data.headline = headlineElement.textContent?.trim();
}
return data;
});
await this.updateJobStatus(jobId, 'running', 80);
return profileData;
}
private async scrapeFacebook(
jobId: string,
page: Page,
profileUrl: string | undefined,
signal: AbortSignal
): Promise<any> {
const url = profileUrl || 'https://facebook.com';
await this.log(jobId, 'info', `Navigating to: ${url}`);
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
await this.updateJobStatus(jobId, 'running', 40);
if (signal.aborted) throw new Error('Job cancelled');
await page.waitForTimeout(2000);
await this.updateJobStatus(jobId, 'running', 60);
const profileData = await page.evaluate(() => {
const doc = document as any;
const win = window as any;
return {
url: win.location.href,
scraped_at: new Date().toISOString(),
title: doc.title,
};
});
await this.updateJobStatus(jobId, 'running', 80);
return profileData;
}
private async scrapeGeneric(
jobId: string,
page: Page,
url: string,
signal: AbortSignal
): Promise<any> {
await this.log(jobId, 'info', `Navigating to: ${url}`);
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
await this.updateJobStatus(jobId, 'running', 50);
if (signal.aborted) throw new Error('Job cancelled');
const data = await page.evaluate(() => {
const doc = document as any;
const win = window as any;
return {
url: win.location.href,
title: doc.title,
scraped_at: new Date().toISOString(),
text_content: doc.body.innerText.substring(0, 5000),
};
});
await this.updateJobStatus(jobId, 'running', 80);
return data;
}
async cancelJob(jobId: string): Promise<void> {
const activeJob = this.activeJobs.get(jobId);
if (activeJob) {
activeJob.abortController.abort();
if (activeJob.context) {
await activeJob.context.close().catch(() => {});
}
await this.updateJobStatus(jobId, 'cancelled');
await this.log(jobId, 'info', 'Job cancelled by user');
this.activeJobs.delete(jobId);
}
}
async shutdown(): Promise<void> {
// Cancel all active jobs
for (const [jobId] of this.activeJobs) {
await this.cancelJob(jobId);
}
// Close browser
if (this.browser) {
await this.browser.close();
this.browser = null;
}
}
}

View File

@@ -0,0 +1,44 @@
import crypto from 'crypto';
const ALGORITHM = 'aes-256-gcm';
const IV_LENGTH = 16;
const AUTH_TAG_LENGTH = 16;
function getEncryptionKey(): Buffer {
const key = process.env.VAULT_ENCRYPTION_KEY;
if (!key || key.length !== 64) {
throw new Error('VAULT_ENCRYPTION_KEY must be a 64-character hex string');
}
return Buffer.from(key, 'hex');
}
export function encrypt(text: string): string {
const iv = crypto.randomBytes(IV_LENGTH);
const cipher = crypto.createCipheriv(ALGORITHM, getEncryptionKey(), iv);
let encrypted = cipher.update(text, 'utf8', 'hex');
encrypted += cipher.final('hex');
const authTag = cipher.getAuthTag();
// Combine IV + AuthTag + Encrypted data
return iv.toString('hex') + authTag.toString('hex') + encrypted;
}
export function decrypt(encryptedData: string): string {
const iv = Buffer.from(encryptedData.slice(0, IV_LENGTH * 2), 'hex');
const authTag = Buffer.from(encryptedData.slice(IV_LENGTH * 2, IV_LENGTH * 2 + AUTH_TAG_LENGTH * 2), 'hex');
const encrypted = encryptedData.slice(IV_LENGTH * 2 + AUTH_TAG_LENGTH * 2);
const decipher = crypto.createDecipheriv(ALGORITHM, getEncryptionKey(), iv);
decipher.setAuthTag(authTag);
let decrypted = decipher.update(encrypted, 'hex', 'utf8');
decrypted += decipher.final('utf8');
return decrypted;
}
export function hashPassword(password: string): string {
return crypto.createHash('sha256').update(password).digest('hex');
}

View File

@@ -0,0 +1,33 @@
import winston from 'winston';
const logFormat = winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format.errors({ stack: true }),
winston.format.printf(({ level, message, timestamp, stack }) => {
return `${timestamp} [${level.toUpperCase()}]: ${stack || message}`;
})
);
export const logger = winston.createLogger({
level: process.env.NODE_ENV === 'production' ? 'info' : 'debug',
format: logFormat,
transports: [
new winston.transports.Console({
format: winston.format.combine(
winston.format.colorize(),
logFormat
),
}),
new winston.transports.File({
filename: 'logs/error.log',
level: 'error',
maxsize: 5242880, // 5MB
maxFiles: 5,
}),
new winston.transports.File({
filename: 'logs/combined.log',
maxsize: 5242880,
maxFiles: 5,
}),
],
});

20
backend/tsconfig.json Normal file
View File

@@ -0,0 +1,20 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"lib": ["ES2022"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

59
docker-compose.yml Normal file
View File

@@ -0,0 +1,59 @@
version: '3.8'
services:
osint-platform:
build:
context: .
dockerfile: Dockerfile
container_name: osint-platform
restart: unless-stopped
ports:
- "3001:3001"
environment:
- NODE_ENV=production
- PORT=3001
# REQUIRED: Set these in Coolify or via .env file
- MONGODB_URI=${MONGODB_URI}
- MASTER_PASSWORD=${MASTER_PASSWORD}
- JWT_SECRET=${JWT_SECRET}
- VAULT_ENCRYPTION_KEY=${VAULT_ENCRYPTION_KEY}
- FRONTEND_URL=${FRONTEND_URL:-http://localhost:3001}
- SESSION_EXPIRY=${SESSION_EXPIRY:-24h}
volumes:
# Logs only (database is MongoDB)
- osint-logs:/app/logs
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3001/api/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
# Security options
security_opt:
- no-new-privileges:true
# Resource limits (adjust based on your needs)
deploy:
resources:
limits:
memory: 2G
reservations:
memory: 512M
# Optional: Local MongoDB for development
# Uncomment if you want to run MongoDB locally via Docker
# mongodb:
# image: mongo:7
# container_name: osint-mongodb
# restart: unless-stopped
# ports:
# - "27017:27017"
# volumes:
# - mongodb-data:/data/db
# environment:
# - MONGO_INITDB_DATABASE=osint_platform
volumes:
osint-logs:
driver: local
# mongodb-data:
# driver: local

18
frontend/index.html Normal file
View File

@@ -0,0 +1,18 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Private OSINT Automation Platform" />
<meta name="theme-color" content="#020617" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet" />
<title>OSINT Platform</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

2848
frontend/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

31
frontend/package.json Normal file
View File

@@ -0,0 +1,31 @@
{
"name": "osint-frontend",
"private": true,
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"preview": "vite preview"
},
"dependencies": {
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "^6.22.3",
"socket.io-client": "^4.7.5",
"lucide-react": "^0.359.0",
"zustand": "^4.5.2",
"date-fns": "^3.6.0"
},
"devDependencies": {
"@types/react": "^18.2.64",
"@types/react-dom": "^18.2.21",
"@vitejs/plugin-react": "^4.2.1",
"autoprefixer": "^10.4.18",
"postcss": "^8.4.35",
"tailwindcss": "^3.4.1",
"typescript": "^5.4.2",
"vite": "^5.1.6"
}
}

View File

@@ -0,0 +1,6 @@
export default {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
}

6
frontend/public/vite.svg Normal file
View File

@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" fill="none">
<rect width="32" height="32" rx="6" fill="#020617"/>
<path d="M16 6L7 11v10l9 5 9-5V11l-9-5z" stroke="#22d3ee" stroke-width="1.5" fill="none"/>
<circle cx="16" cy="16" r="4" fill="#22d3ee" opacity="0.3"/>
<path d="M16 12v8M12 16h8" stroke="#22d3ee" stroke-width="1.5" stroke-linecap="round"/>
</svg>

After

Width:  |  Height:  |  Size: 387 B

36
frontend/src/App.tsx Normal file
View File

@@ -0,0 +1,36 @@
import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom';
import { useAuthStore } from './stores/authStore';
import { LoginPage } from './pages/LoginPage';
import { Dashboard } from './pages/Dashboard';
function ProtectedRoute({ children }: { children: React.ReactNode }) {
const isAuthenticated = useAuthStore((state) => state.isAuthenticated);
if (!isAuthenticated) {
return <Navigate to="/login" replace />;
}
return <>{children}</>;
}
function App() {
return (
<BrowserRouter future={{ v7_startTransition: true, v7_relativeSplatPath: true }}>
<div className="min-h-screen bg-slate-950">
<Routes>
<Route path="/login" element={<LoginPage />} />
<Route
path="/*"
element={
<ProtectedRoute>
<Dashboard />
</ProtectedRoute>
}
/>
</Routes>
</div>
</BrowserRouter>
);
}
export default App;

View File

@@ -0,0 +1,161 @@
import { useState } from 'react';
import { useTargetStore } from '../stores/targetStore';
import { X, Globe, Twitter, Instagram, Linkedin, Facebook } from 'lucide-react';
interface AddProfileModalProps {
onClose: () => void;
}
const PLATFORMS = [
{ id: 'x', name: 'X (Twitter)', icon: Twitter },
{ id: 'instagram', name: 'Instagram', icon: Instagram },
{ id: 'linkedin', name: 'LinkedIn', icon: Linkedin },
{ id: 'facebook', name: 'Facebook', icon: Facebook },
{ id: 'other', name: 'Other', icon: Globe },
];
export function AddProfileModal({ onClose }: AddProfileModalProps) {
const { selectedTarget, addProfile } = useTargetStore();
const [platform, setPlatform] = useState('x');
const [username, setUsername] = useState('');
const [profileUrl, setProfileUrl] = useState('');
const [loading, setLoading] = useState(false);
const [error, setError] = useState('');
if (!selectedTarget) {
return null;
}
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
if (!platform) return;
setLoading(true);
setError('');
try {
await addProfile(
selectedTarget.id,
platform,
username.trim() || undefined,
profileUrl.trim() || undefined
);
onClose();
} catch (err: any) {
setError(err.message || 'Failed to add profile');
} finally {
setLoading(false);
}
};
return (
<div className="fixed inset-0 z-50 flex items-center justify-center">
{/* Backdrop */}
<div
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
onClick={onClose}
/>
{/* Modal */}
<div className="relative w-full max-w-md mx-4 bg-slate-900 border border-slate-800 rounded-xl shadow-2xl">
{/* Header */}
<div className="flex items-center justify-between p-4 border-b border-slate-800">
<div className="flex items-center gap-2">
<Globe className="w-5 h-5 text-accent-cyan" />
<div>
<h2 className="text-lg font-semibold text-slate-100">Add Profile</h2>
<p className="text-xs text-slate-500">for {selectedTarget.name}</p>
</div>
</div>
<button
onClick={onClose}
className="p-1.5 rounded-lg text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
>
<X className="w-5 h-5" />
</button>
</div>
{/* Form */}
<form onSubmit={handleSubmit} className="p-4 space-y-4">
{error && (
<div className="p-3 rounded-lg bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
{error}
</div>
)}
<div>
<label className="block text-sm font-medium text-slate-400 mb-2">
Platform *
</label>
<div className="grid grid-cols-5 gap-2">
{PLATFORMS.map((p) => {
const Icon = p.icon;
return (
<button
key={p.id}
type="button"
onClick={() => setPlatform(p.id)}
className={`flex flex-col items-center gap-1 p-3 rounded-lg border transition-colors ${
platform === p.id
? 'border-accent-cyan bg-accent-cyan/10 text-accent-cyan'
: 'border-slate-700 text-slate-400 hover:border-slate-600 hover:text-slate-300'
}`}
>
<Icon className="w-5 h-5" />
<span className="text-[10px]">{p.id === 'x' ? 'X' : p.id.slice(0, 2).toUpperCase()}</span>
</button>
);
})}
</div>
</div>
<div>
<label htmlFor="username" className="block text-sm font-medium text-slate-400 mb-2">
Username
</label>
<input
id="username"
type="text"
value={username}
onChange={(e) => setUsername(e.target.value)}
className="input-field w-full font-mono"
placeholder="@username"
/>
</div>
<div>
<label htmlFor="profileUrl" className="block text-sm font-medium text-slate-400 mb-2">
Profile URL
</label>
<input
id="profileUrl"
type="url"
value={profileUrl}
onChange={(e) => setProfileUrl(e.target.value)}
className="input-field w-full font-mono text-sm"
placeholder="https://..."
/>
</div>
<div className="flex gap-3 pt-2">
<button
type="button"
onClick={onClose}
className="btn-secondary flex-1"
disabled={loading}
>
Cancel
</button>
<button
type="submit"
className="btn-primary flex-1"
disabled={!platform || loading}
>
{loading ? 'Adding...' : 'Add Profile'}
</button>
</div>
</form>
</div>
</div>
);
}

View File

@@ -0,0 +1,190 @@
import { useState } from 'react';
import { useScraperStore } from '../stores/scraperStore';
import { X, Key, AlertCircle, HelpCircle } from 'lucide-react';
interface AddSessionModalProps {
onClose: () => void;
}
const PLATFORMS = [
{ id: 'x', name: 'X (Twitter)' },
{ id: 'instagram', name: 'Instagram' },
{ id: 'linkedin', name: 'LinkedIn' },
{ id: 'facebook', name: 'Facebook' },
];
export function AddSessionModal({ onClose }: AddSessionModalProps) {
const { addSession } = useScraperStore();
const [platform, setPlatform] = useState('x');
const [sessionName, setSessionName] = useState('');
const [cookiesJson, setCookiesJson] = useState('');
const [userAgent, setUserAgent] = useState('');
const [loading, setLoading] = useState(false);
const [error, setError] = useState('');
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
if (!platform || !sessionName.trim() || !cookiesJson.trim()) {
setError('Please fill in all required fields');
return;
}
// Parse cookies JSON
let cookies: any[];
try {
cookies = JSON.parse(cookiesJson);
if (!Array.isArray(cookies)) {
throw new Error('Cookies must be an array');
}
} catch (err) {
setError('Invalid cookies JSON. Please provide a valid array of cookie objects.');
return;
}
setLoading(true);
setError('');
try {
await addSession({
platform,
session_name: sessionName.trim(),
cookies,
user_agent: userAgent.trim() || undefined,
});
onClose();
} catch (err: any) {
setError(err.message || 'Failed to add session');
} finally {
setLoading(false);
}
};
return (
<div className="fixed inset-0 z-50 flex items-center justify-center">
{/* Backdrop */}
<div
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
onClick={onClose}
/>
{/* Modal */}
<div className="relative w-full max-w-lg mx-4 bg-slate-900 border border-slate-800 rounded-xl shadow-2xl">
{/* Header */}
<div className="flex items-center justify-between p-4 border-b border-slate-800">
<div className="flex items-center gap-2">
<Key className="w-5 h-5 text-accent-cyan" />
<h2 className="text-lg font-semibold text-slate-100">Add Session</h2>
</div>
<button
onClick={onClose}
className="p-1.5 rounded-lg text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
>
<X className="w-5 h-5" />
</button>
</div>
{/* Form */}
<form onSubmit={handleSubmit} className="p-4 space-y-4">
{error && (
<div className="flex items-start gap-2 p-3 rounded-lg bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
<AlertCircle className="w-4 h-4 mt-0.5 flex-shrink-0" />
<span>{error}</span>
</div>
)}
{/* Info box */}
<div className="flex items-start gap-2 p-3 rounded-lg bg-accent-cyan/10 border border-accent-cyan/20 text-accent-cyan text-xs">
<HelpCircle className="w-4 h-4 mt-0.5 flex-shrink-0" />
<div>
<p className="font-medium mb-1">How to get cookies:</p>
<ol className="list-decimal list-inside text-accent-cyan/80 space-y-0.5">
<li>Log into the platform in your browser</li>
<li>Open DevTools (F12) Application Cookies</li>
<li>Export cookies as JSON array</li>
<li>Or use a browser extension like "Cookie Editor"</li>
</ol>
</div>
</div>
<div className="grid grid-cols-2 gap-4">
<div>
<label htmlFor="platform" className="block text-sm font-medium text-slate-400 mb-2">
Platform *
</label>
<select
id="platform"
value={platform}
onChange={(e) => setPlatform(e.target.value)}
className="input-field w-full"
>
{PLATFORMS.map((p) => (
<option key={p.id} value={p.id}>{p.name}</option>
))}
</select>
</div>
<div>
<label htmlFor="sessionName" className="block text-sm font-medium text-slate-400 mb-2">
Session Name *
</label>
<input
id="sessionName"
type="text"
value={sessionName}
onChange={(e) => setSessionName(e.target.value)}
className="input-field w-full"
placeholder="My Account"
/>
</div>
</div>
<div>
<label htmlFor="cookies" className="block text-sm font-medium text-slate-400 mb-2">
Cookies JSON *
</label>
<textarea
id="cookies"
value={cookiesJson}
onChange={(e) => setCookiesJson(e.target.value)}
className="input-field w-full h-32 resize-none font-mono text-xs"
placeholder='[{"name": "auth_token", "value": "...", "domain": ".twitter.com"}]'
/>
</div>
<div>
<label htmlFor="userAgent" className="block text-sm font-medium text-slate-400 mb-2">
User Agent <span className="text-slate-600">(optional)</span>
</label>
<input
id="userAgent"
type="text"
value={userAgent}
onChange={(e) => setUserAgent(e.target.value)}
className="input-field w-full font-mono text-xs"
placeholder="Mozilla/5.0 ..."
/>
</div>
<div className="flex gap-3 pt-2">
<button
type="button"
onClick={onClose}
className="btn-secondary flex-1"
disabled={loading}
>
Cancel
</button>
<button
type="submit"
className="btn-primary flex-1"
disabled={!platform || !sessionName.trim() || !cookiesJson.trim() || loading}
>
{loading ? 'Saving...' : 'Save Session'}
</button>
</div>
</form>
</div>
</div>
);
}

View File

@@ -0,0 +1,114 @@
import { useState } from 'react';
import { useTargetStore } from '../stores/targetStore';
import { X, UserPlus } from 'lucide-react';
interface AddTargetModalProps {
onClose: () => void;
}
export function AddTargetModal({ onClose }: AddTargetModalProps) {
const { createTarget } = useTargetStore();
const [name, setName] = useState('');
const [notes, setNotes] = useState('');
const [loading, setLoading] = useState(false);
const [error, setError] = useState('');
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
if (!name.trim()) return;
setLoading(true);
setError('');
try {
await createTarget(name.trim(), notes.trim() || undefined);
onClose();
} catch (err: any) {
setError(err.message || 'Failed to create target');
} finally {
setLoading(false);
}
};
return (
<div className="fixed inset-0 z-50 flex items-center justify-center">
{/* Backdrop */}
<div
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
onClick={onClose}
/>
{/* Modal */}
<div className="relative w-full max-w-md mx-4 bg-slate-900 border border-slate-800 rounded-xl shadow-2xl">
{/* Header */}
<div className="flex items-center justify-between p-4 border-b border-slate-800">
<div className="flex items-center gap-2">
<UserPlus className="w-5 h-5 text-accent-cyan" />
<h2 className="text-lg font-semibold text-slate-100">New Target</h2>
</div>
<button
onClick={onClose}
className="p-1.5 rounded-lg text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
>
<X className="w-5 h-5" />
</button>
</div>
{/* Form */}
<form onSubmit={handleSubmit} className="p-4 space-y-4">
{error && (
<div className="p-3 rounded-lg bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
{error}
</div>
)}
<div>
<label htmlFor="name" className="block text-sm font-medium text-slate-400 mb-2">
Target Name *
</label>
<input
id="name"
type="text"
value={name}
onChange={(e) => setName(e.target.value)}
className="input-field w-full"
placeholder="e.g., John Doe"
autoFocus
/>
</div>
<div>
<label htmlFor="notes" className="block text-sm font-medium text-slate-400 mb-2">
Notes
</label>
<textarea
id="notes"
value={notes}
onChange={(e) => setNotes(e.target.value)}
className="input-field w-full h-24 resize-none"
placeholder="Additional information about this target..."
/>
</div>
<div className="flex gap-3 pt-2">
<button
type="button"
onClick={onClose}
className="btn-secondary flex-1"
disabled={loading}
>
Cancel
</button>
<button
type="submit"
className="btn-primary flex-1"
disabled={!name.trim() || loading}
>
{loading ? 'Creating...' : 'Create Target'}
</button>
</div>
</form>
</div>
</div>
);
}

View File

@@ -0,0 +1,86 @@
import { Shield, LogOut, Plus, Key, Activity } from 'lucide-react';
import { useAuthStore } from '../stores/authStore';
import { useScraperStore } from '../stores/scraperStore';
import { useNavigate } from 'react-router-dom';
interface HeaderProps {
onAddTarget: () => void;
onAddSession: () => void;
}
export function Header({ onAddTarget, onAddSession }: HeaderProps) {
const { logout } = useAuthStore();
const { activeJobs, sessions } = useScraperStore();
const navigate = useNavigate();
const handleLogout = () => {
logout();
navigate('/login');
};
const runningJobs = activeJobs.filter(j => j.status === 'running').length;
const activeSessions = sessions.filter(s => s.status === 'active').length;
return (
<header className="h-14 flex-shrink-0 bg-slate-900 border-b border-slate-800 px-4 flex items-center justify-between">
{/* Logo */}
<div className="flex items-center gap-3">
<div className="flex items-center justify-center w-8 h-8 rounded-lg bg-gradient-to-br from-accent-cyan/20 to-accent-violet/20 border border-accent-cyan/30">
<Shield className="w-4 h-4 text-accent-cyan" />
</div>
<div>
<h1 className="text-sm font-semibold text-slate-100">OSINT Platform</h1>
<p className="text-[10px] text-slate-500 font-mono uppercase tracking-wider">Intelligence Automation</p>
</div>
</div>
{/* Status indicators */}
<div className="flex items-center gap-4">
{/* Running jobs indicator */}
<div className="flex items-center gap-2 px-3 py-1.5 rounded-full bg-slate-800/50 border border-slate-700/50">
<Activity className={`w-3.5 h-3.5 ${runningJobs > 0 ? 'text-accent-emerald animate-pulse' : 'text-slate-500'}`} />
<span className="text-xs font-medium text-slate-400">
{runningJobs > 0 ? `${runningJobs} Active` : 'Idle'}
</span>
</div>
{/* Sessions indicator */}
<div className="flex items-center gap-2 px-3 py-1.5 rounded-full bg-slate-800/50 border border-slate-700/50">
<Key className={`w-3.5 h-3.5 ${activeSessions > 0 ? 'text-accent-cyan' : 'text-slate-500'}`} />
<span className="text-xs font-medium text-slate-400">
{activeSessions} Sessions
</span>
</div>
</div>
{/* Actions */}
<div className="flex items-center gap-2">
<button
onClick={onAddSession}
className="btn-secondary flex items-center gap-1.5 text-xs py-1.5"
>
<Key className="w-3.5 h-3.5" />
<span>Add Session</span>
</button>
<button
onClick={onAddTarget}
className="btn-primary flex items-center gap-1.5 text-xs py-1.5"
>
<Plus className="w-3.5 h-3.5" />
<span>New Target</span>
</button>
<div className="w-px h-6 bg-slate-700 mx-2" />
<button
onClick={handleLogout}
className="p-2 rounded text-slate-400 hover:text-slate-200 hover:bg-slate-800 transition-colors"
title="Logout"
>
<LogOut className="w-4 h-4" />
</button>
</div>
</header>
);
}

View File

@@ -0,0 +1,238 @@
import { useState } from 'react';
import { useTargetStore } from '../stores/targetStore';
import { useScraperStore } from '../stores/scraperStore';
import {
Globe, Twitter, Instagram, Linkedin, Facebook,
ExternalLink, Play, Trash2, ChevronDown, ChevronUp,
Database, FileJson, Copy, CheckCircle, AlertCircle
} from 'lucide-react';
import { formatDistanceToNow } from 'date-fns';
const PLATFORM_ICONS: Record<string, any> = {
twitter: Twitter,
x: Twitter,
instagram: Instagram,
linkedin: Linkedin,
facebook: Facebook,
default: Globe,
};
const PLATFORM_COLORS: Record<string, string> = {
twitter: 'text-sky-400',
x: 'text-slate-300',
instagram: 'text-pink-400',
linkedin: 'text-blue-400',
facebook: 'text-blue-500',
default: 'text-slate-400',
};
export function MainDataFeed() {
const { selectedTarget, deleteProfile } = useTargetStore();
const { startJob } = useScraperStore();
const [expandedProfile, setExpandedProfile] = useState<string | null>(null);
const [scraping, setScraping] = useState<string | null>(null);
const [copied, setCopied] = useState<string | null>(null);
if (!selectedTarget) {
return (
<div className="h-full flex items-center justify-center bg-slate-950">
<div className="text-center">
<Database className="w-16 h-16 text-slate-800 mx-auto mb-4" />
<h2 className="text-lg font-medium text-slate-400 mb-2">No Target Selected</h2>
<p className="text-sm text-slate-600 max-w-xs">
Select a target from the left panel to view their social media profiles and scraped data.
</p>
</div>
</div>
);
}
const profiles = selectedTarget.profiles || [];
const handleScrape = async (profileId: string, platform: string, profileUrl?: string) => {
setScraping(profileId);
try {
await startJob({
platform,
profileUrl,
targetId: selectedTarget.id,
profileId,
});
} catch (err) {
console.error('Failed to start scrape:', err);
} finally {
setScraping(null);
}
};
const handleDelete = async (profileId: string) => {
if (confirm('Delete this profile?')) {
await deleteProfile(selectedTarget.id, profileId);
}
};
const copyData = (data: any, id: string) => {
navigator.clipboard.writeText(JSON.stringify(data, null, 2));
setCopied(id);
setTimeout(() => setCopied(null), 2000);
};
return (
<div className="h-full flex flex-col bg-slate-950">
{/* Header */}
<div className="p-4 border-b border-slate-800">
<div className="flex items-center justify-between">
<div>
<h2 className="text-lg font-semibold text-slate-100">{selectedTarget.name}</h2>
<p className="text-xs text-slate-500 font-mono mt-0.5">
ID: {selectedTarget.id.slice(0, 8)}...
</p>
</div>
<div className="flex items-center gap-2">
<span className="text-xs text-slate-500">
{profiles.length} profile{profiles.length !== 1 ? 's' : ''}
</span>
</div>
</div>
{selectedTarget.notes && (
<p className="text-sm text-slate-400 mt-3 p-3 bg-slate-900 rounded border border-slate-800">
{selectedTarget.notes}
</p>
)}
</div>
{/* Profiles */}
<div className="flex-1 overflow-y-auto p-4">
{profiles.length === 0 ? (
<div className="text-center py-12">
<Globe className="w-12 h-12 text-slate-700 mx-auto mb-3" />
<p className="text-sm text-slate-500">No profiles linked</p>
<p className="text-xs text-slate-600 mt-1">Add social media profiles to start collecting data</p>
</div>
) : (
<div className="space-y-3">
{profiles.map(profile => {
const Icon = PLATFORM_ICONS[profile.platform.toLowerCase()] || PLATFORM_ICONS.default;
const colorClass = PLATFORM_COLORS[profile.platform.toLowerCase()] || PLATFORM_COLORS.default;
const isExpanded = expandedProfile === profile.id;
const profileData = profile.profile_data ? JSON.parse(profile.profile_data) : null;
return (
<div key={profile.id} className="card border-slate-800">
{/* Profile header */}
<div className="p-4">
<div className="flex items-start justify-between">
<div className="flex items-center gap-3">
<div className={`p-2 rounded-lg bg-slate-800 ${colorClass}`}>
<Icon className="w-5 h-5" />
</div>
<div>
<div className="flex items-center gap-2">
<span className="font-medium text-slate-200">
{profile.username || profile.platform}
</span>
<span className="text-xs text-slate-600 font-mono uppercase">
{profile.platform}
</span>
</div>
{profile.profile_url && (
<a
href={profile.profile_url}
target="_blank"
rel="noopener noreferrer"
className="text-xs text-slate-500 hover:text-accent-cyan flex items-center gap-1 mt-0.5"
>
<span className="truncate max-w-[200px]">{profile.profile_url}</span>
<ExternalLink className="w-3 h-3" />
</a>
)}
</div>
</div>
<div className="flex items-center gap-1">
<button
onClick={() => handleScrape(profile.id, profile.platform, profile.profile_url || undefined)}
disabled={scraping === profile.id}
className="p-2 rounded hover:bg-slate-800 text-accent-cyan transition-colors disabled:opacity-50"
title="Start scrape"
>
{scraping === profile.id ? (
<div className="w-4 h-4 border-2 border-accent-cyan border-t-transparent rounded-full animate-spin" />
) : (
<Play className="w-4 h-4" />
)}
</button>
<button
onClick={() => handleDelete(profile.id)}
className="p-2 rounded hover:bg-slate-800 text-slate-500 hover:text-accent-rose transition-colors"
title="Delete profile"
>
<Trash2 className="w-4 h-4" />
</button>
</div>
</div>
{/* Scraped status */}
<div className="flex items-center gap-4 mt-3 pt-3 border-t border-slate-800">
{profile.last_scraped ? (
<div className="flex items-center gap-1.5 text-xs text-accent-emerald">
<CheckCircle className="w-3.5 h-3.5" />
<span>Last scraped {formatDistanceToNow(new Date(profile.last_scraped), { addSuffix: true })}</span>
</div>
) : (
<div className="flex items-center gap-1.5 text-xs text-slate-500">
<AlertCircle className="w-3.5 h-3.5" />
<span>Never scraped</span>
</div>
)}
{profileData && (
<button
onClick={() => setExpandedProfile(isExpanded ? null : profile.id)}
className="flex items-center gap-1 text-xs text-slate-400 hover:text-slate-200"
>
<FileJson className="w-3.5 h-3.5" />
<span>View Data</span>
{isExpanded ? <ChevronUp className="w-3.5 h-3.5" /> : <ChevronDown className="w-3.5 h-3.5" />}
</button>
)}
</div>
</div>
{/* Expanded data view */}
{isExpanded && profileData && (
<div className="border-t border-slate-800 p-4 bg-slate-900/50">
<div className="flex items-center justify-between mb-3">
<span className="text-xs font-medium text-slate-400">Scraped Data</span>
<button
onClick={() => copyData(profileData, profile.id)}
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300"
>
{copied === profile.id ? (
<>
<CheckCircle className="w-3.5 h-3.5 text-accent-emerald" />
<span className="text-accent-emerald">Copied!</span>
</>
) : (
<>
<Copy className="w-3.5 h-3.5" />
<span>Copy JSON</span>
</>
)}
</button>
</div>
<pre className="text-xs text-slate-300 font-mono bg-slate-950 p-3 rounded overflow-x-auto max-h-64 overflow-y-auto">
{JSON.stringify(profileData, null, 2)}
</pre>
</div>
)}
</div>
);
})}
</div>
)}
</div>
</div>
);
}

View File

@@ -0,0 +1,288 @@
import { useState } from 'react';
import { useScraperStore, ScraperJob, Session } from '../stores/scraperStore';
import {
Activity, Key, Clock, CheckCircle, XCircle,
StopCircle, ChevronDown, Terminal, RefreshCw
} from 'lucide-react';
import { formatDistanceToNow, format } from 'date-fns';
type Tab = 'jobs' | 'sessions';
export function ScraperPanel() {
const [activeTab, setActiveTab] = useState<Tab>('jobs');
const { jobs, sessions, activeJobs, fetchJobs, fetchSessions } = useScraperStore();
return (
<div className="h-full flex flex-col bg-slate-900/50">
{/* Tabs */}
<div className="flex border-b border-slate-800">
<button
onClick={() => setActiveTab('jobs')}
className={`flex-1 px-4 py-3 text-xs font-medium flex items-center justify-center gap-2 transition-colors ${
activeTab === 'jobs'
? 'text-accent-cyan border-b-2 border-accent-cyan bg-slate-800/30'
: 'text-slate-400 hover:text-slate-200'
}`}
>
<Activity className="w-3.5 h-3.5" />
Jobs
{activeJobs.length > 0 && (
<span className="px-1.5 py-0.5 rounded-full bg-accent-cyan/20 text-accent-cyan text-[10px]">
{activeJobs.length}
</span>
)}
</button>
<button
onClick={() => setActiveTab('sessions')}
className={`flex-1 px-4 py-3 text-xs font-medium flex items-center justify-center gap-2 transition-colors ${
activeTab === 'sessions'
? 'text-accent-cyan border-b-2 border-accent-cyan bg-slate-800/30'
: 'text-slate-400 hover:text-slate-200'
}`}
>
<Key className="w-3.5 h-3.5" />
Sessions
<span className="px-1.5 py-0.5 rounded-full bg-slate-700 text-slate-300 text-[10px]">
{sessions.length}
</span>
</button>
</div>
{/* Content */}
<div className="flex-1 overflow-y-auto">
{activeTab === 'jobs' ? (
<JobsList jobs={jobs} onRefresh={fetchJobs} />
) : (
<SessionsList sessions={sessions} onRefresh={fetchSessions} />
)}
</div>
</div>
);
}
function JobsList({ jobs, onRefresh }: { jobs: ScraperJob[]; onRefresh: () => void }) {
const { cancelJob, logs } = useScraperStore();
const [expandedJob, setExpandedJob] = useState<string | null>(null);
const getStatusIcon = (status: string) => {
switch (status) {
case 'running':
return <div className="w-3 h-3 border-2 border-accent-cyan border-t-transparent rounded-full animate-spin" />;
case 'completed':
return <CheckCircle className="w-4 h-4 text-accent-emerald" />;
case 'failed':
return <XCircle className="w-4 h-4 text-accent-rose" />;
case 'cancelled':
return <StopCircle className="w-4 h-4 text-slate-500" />;
default:
return <Clock className="w-4 h-4 text-accent-amber" />;
}
};
const getStatusColor = (status: string) => {
switch (status) {
case 'running': return 'border-accent-cyan/30 bg-accent-cyan/5';
case 'completed': return 'border-accent-emerald/30 bg-accent-emerald/5';
case 'failed': return 'border-accent-rose/30 bg-accent-rose/5';
default: return 'border-slate-700 bg-slate-800/50';
}
};
return (
<div className="p-3 space-y-2">
<div className="flex items-center justify-between mb-2">
<span className="text-xs text-slate-500">{jobs.length} total jobs</span>
<button
onClick={onRefresh}
className="p-1.5 rounded text-slate-400 hover:text-slate-200 hover:bg-slate-800"
>
<RefreshCw className="w-3.5 h-3.5" />
</button>
</div>
{jobs.length === 0 ? (
<div className="text-center py-8">
<Activity className="w-10 h-10 text-slate-700 mx-auto mb-2" />
<p className="text-xs text-slate-500">No scraper jobs yet</p>
</div>
) : (
jobs.slice(0, 20).map(job => {
const isExpanded = expandedJob === job.id;
const jobLogs = logs.get(job.id) || [];
return (
<div
key={job.id}
className={`rounded-lg border ${getStatusColor(job.status)} transition-colors`}
>
<button
onClick={() => setExpandedJob(isExpanded ? null : job.id)}
className="w-full p-3 text-left"
>
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
{getStatusIcon(job.status)}
<span className="text-sm font-medium text-slate-200 capitalize">
{job.platform}
</span>
</div>
<ChevronDown className={`w-4 h-4 text-slate-500 transition-transform ${isExpanded ? 'rotate-180' : ''}`} />
</div>
<div className="flex items-center gap-3 mt-2">
{job.status === 'running' && (
<div className="flex-1">
<div className="h-1 bg-slate-700 rounded-full overflow-hidden">
<div
className="h-full bg-accent-cyan transition-all"
style={{ width: `${job.progress}%` }}
/>
</div>
</div>
)}
<span className="text-[10px] text-slate-500 font-mono">
{formatDistanceToNow(new Date(job.created_at), { addSuffix: true })}
</span>
</div>
</button>
{isExpanded && (
<div className="px-3 pb-3 border-t border-slate-700/50 mt-2 pt-2">
<div className="space-y-1 text-xs">
<div className="flex justify-between">
<span className="text-slate-500">Job ID:</span>
<span className="text-slate-300 font-mono">{job.id.slice(0, 8)}...</span>
</div>
{job.target_name && (
<div className="flex justify-between">
<span className="text-slate-500">Target:</span>
<span className="text-slate-300">{job.target_name}</span>
</div>
)}
{job.error && (
<div className="mt-2 p-2 bg-accent-rose/10 rounded text-accent-rose text-xs">
{job.error}
</div>
)}
{job.status === 'running' && (
<button
onClick={() => cancelJob(job.id)}
className="w-full mt-2 py-1.5 rounded border border-accent-rose/30 text-accent-rose text-xs hover:bg-accent-rose/10"
>
Cancel Job
</button>
)}
{/* Logs */}
{jobLogs.length > 0 && (
<div className="mt-3">
<div className="flex items-center gap-1 text-slate-500 mb-2">
<Terminal className="w-3 h-3" />
<span>Logs</span>
</div>
<div className="bg-slate-950 rounded p-2 max-h-32 overflow-y-auto font-mono text-[10px] space-y-0.5">
{jobLogs.slice(-10).map((log, i) => (
<div key={i} className={`${log.level === 'error' ? 'text-accent-rose' : 'text-slate-400'}`}>
<span className="text-slate-600">[{format(new Date(log.timestamp), 'HH:mm:ss')}]</span>{' '}
{log.message}
</div>
))}
</div>
</div>
)}
</div>
</div>
)}
</div>
);
})
)}
</div>
);
}
function SessionsList({ sessions, onRefresh }: { sessions: Session[]; onRefresh: () => void }) {
const { deleteSession } = useScraperStore();
const getStatusIndicator = (status: string) => {
switch (status) {
case 'active':
return <div className="status-indicator status-active" />;
case 'expired':
return <div className="status-indicator status-warning" />;
default:
return <div className="status-indicator status-error" />;
}
};
return (
<div className="p-3 space-y-2">
<div className="flex items-center justify-between mb-2">
<span className="text-xs text-slate-500">Session Vault</span>
<button
onClick={onRefresh}
className="p-1.5 rounded text-slate-400 hover:text-slate-200 hover:bg-slate-800"
>
<RefreshCw className="w-3.5 h-3.5" />
</button>
</div>
{sessions.length === 0 ? (
<div className="text-center py-8">
<Key className="w-10 h-10 text-slate-700 mx-auto mb-2" />
<p className="text-xs text-slate-500">No sessions stored</p>
<p className="text-[10px] text-slate-600 mt-1">Add session cookies to enable authenticated scraping</p>
</div>
) : (
sessions.map(session => (
<div key={session.id} className="p-3 rounded-lg border border-slate-800 bg-slate-800/30">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
{getStatusIndicator(session.status)}
<span className="text-sm font-medium text-slate-200 capitalize">
{session.platform}
</span>
</div>
<span className={`text-[10px] px-2 py-0.5 rounded-full ${
session.status === 'active'
? 'bg-accent-emerald/20 text-accent-emerald'
: session.status === 'expired'
? 'bg-accent-amber/20 text-accent-amber'
: 'bg-accent-rose/20 text-accent-rose'
}`}>
{session.status}
</span>
</div>
<div className="mt-2 space-y-1 text-xs">
<div className="flex justify-between">
<span className="text-slate-500">Name:</span>
<span className="text-slate-300">{session.session_name}</span>
</div>
{session.last_validated && (
<div className="flex justify-between">
<span className="text-slate-500">Validated:</span>
<span className="text-slate-400 font-mono text-[10px]">
{formatDistanceToNow(new Date(session.last_validated), { addSuffix: true })}
</span>
</div>
)}
</div>
<button
onClick={() => {
if (confirm(`Delete ${session.platform} session?`)) {
deleteSession(session.id);
}
}}
className="w-full mt-3 py-1.5 rounded border border-slate-700 text-slate-400 text-xs hover:border-accent-rose/30 hover:text-accent-rose hover:bg-accent-rose/5 transition-colors"
>
Remove Session
</button>
</div>
))
)}
</div>
);
}

View File

@@ -0,0 +1,199 @@
import { useState } from 'react';
import { useTargetStore, Target } from '../stores/targetStore';
import {
Users, Search, ChevronRight, Plus, MoreHorizontal,
Trash2, Edit2, UserCircle2
} from 'lucide-react';
import { formatDistanceToNow } from 'date-fns';
interface TargetListProps {
onAddProfile: () => void;
}
export function TargetList({ onAddProfile }: TargetListProps) {
const { targets, selectedTarget, selectTarget, fetchTarget, loading } = useTargetStore();
const [search, setSearch] = useState('');
const [contextMenu, setContextMenu] = useState<string | null>(null);
const filteredTargets = targets.filter(t =>
t.name.toLowerCase().includes(search.toLowerCase())
);
const handleSelectTarget = async (target: Target) => {
selectTarget(target);
await fetchTarget(target.id);
};
return (
<div className="h-full flex flex-col bg-slate-900/50">
{/* Header */}
<div className="p-3 border-b border-slate-800">
<div className="flex items-center justify-between mb-3">
<div className="flex items-center gap-2">
<Users className="w-4 h-4 text-slate-400" />
<span className="text-sm font-medium text-slate-300">Targets</span>
</div>
<span className="text-xs text-slate-500 font-mono">{targets.length}</span>
</div>
<div className="relative">
<Search className="absolute left-2.5 top-1/2 -translate-y-1/2 w-3.5 h-3.5 text-slate-500" />
<input
type="text"
placeholder="Search targets..."
value={search}
onChange={(e) => setSearch(e.target.value)}
className="input-field w-full pl-8 py-1.5 text-xs"
/>
</div>
</div>
{/* Target list */}
<div className="flex-1 overflow-y-auto">
{loading && targets.length === 0 ? (
<div className="p-4 space-y-2">
{[1, 2, 3].map(i => (
<div key={i} className="h-16 shimmer rounded" />
))}
</div>
) : filteredTargets.length === 0 ? (
<div className="p-4 text-center">
<UserCircle2 className="w-10 h-10 text-slate-700 mx-auto mb-2" />
<p className="text-sm text-slate-500">
{search ? 'No targets found' : 'No targets yet'}
</p>
<p className="text-xs text-slate-600 mt-1">
Add a target to get started
</p>
</div>
) : (
<div className="p-2 space-y-1">
{filteredTargets.map(target => (
<TargetItem
key={target.id}
target={target}
isSelected={selectedTarget?.id === target.id}
onSelect={() => handleSelectTarget(target)}
showContextMenu={contextMenu === target.id}
onToggleContext={() => setContextMenu(contextMenu === target.id ? null : target.id)}
onAddProfile={onAddProfile}
/>
))}
</div>
)}
</div>
</div>
);
}
interface TargetItemProps {
target: Target;
isSelected: boolean;
onSelect: () => void;
showContextMenu: boolean;
onToggleContext: () => void;
onAddProfile: () => void;
}
function TargetItem({
target,
isSelected,
onSelect,
showContextMenu,
onToggleContext,
onAddProfile
}: TargetItemProps) {
const { deleteTarget } = useTargetStore();
const handleDelete = async (e: React.MouseEvent) => {
e.stopPropagation();
if (confirm(`Delete target "${target.name}"?`)) {
await deleteTarget(target.id);
}
};
return (
<div className="relative">
<button
onClick={onSelect}
className={`w-full text-left p-3 rounded-lg transition-all ${
isSelected
? 'bg-slate-800 border border-accent-cyan/30'
: 'hover:bg-slate-800/50 border border-transparent'
}`}
>
<div className="flex items-start justify-between">
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<span className="text-sm font-medium text-slate-200 truncate">
{target.name}
</span>
{isSelected && (
<ChevronRight className="w-3.5 h-3.5 text-accent-cyan flex-shrink-0" />
)}
</div>
<div className="flex items-center gap-2 mt-1">
<span className="text-xs text-slate-500">
{target.profile_count || 0} profiles
</span>
<span className="text-xs text-slate-600"></span>
<span className="text-[10px] text-slate-600 font-mono">
{target.updated_at && !isNaN(new Date(target.updated_at).getTime())
? formatDistanceToNow(new Date(target.updated_at), { addSuffix: true })
: 'just now'}
</span>
</div>
</div>
<button
onClick={(e) => {
e.stopPropagation();
onToggleContext();
}}
className="p-1 rounded hover:bg-slate-700 text-slate-500 hover:text-slate-300"
>
<MoreHorizontal className="w-4 h-4" />
</button>
</div>
</button>
{/* Context menu */}
{showContextMenu && (
<>
<div
className="fixed inset-0 z-10"
onClick={onToggleContext}
/>
<div className="absolute right-2 top-10 z-20 w-40 py-1 bg-slate-800 border border-slate-700 rounded-lg shadow-xl">
<button
onClick={(e) => {
e.stopPropagation();
onSelect();
onAddProfile();
onToggleContext();
}}
className="w-full px-3 py-2 text-left text-xs text-slate-300 hover:bg-slate-700 flex items-center gap-2"
>
<Plus className="w-3.5 h-3.5" />
Add Profile
</button>
<button
className="w-full px-3 py-2 text-left text-xs text-slate-300 hover:bg-slate-700 flex items-center gap-2"
>
<Edit2 className="w-3.5 h-3.5" />
Edit Target
</button>
<div className="my-1 border-t border-slate-700" />
<button
onClick={handleDelete}
className="w-full px-3 py-2 text-left text-xs text-accent-rose hover:bg-accent-rose/10 flex items-center gap-2"
>
<Trash2 className="w-3.5 h-3.5" />
Delete Target
</button>
</div>
</>
)}
</div>
);
}

125
frontend/src/index.css Normal file
View File

@@ -0,0 +1,125 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
/* Base styles */
:root {
--color-bg-primary: #020617;
--color-bg-secondary: #0f172a;
--color-bg-tertiary: #1e293b;
--color-text-primary: #f1f5f9;
--color-text-secondary: #94a3b8;
--color-accent: #22d3ee;
}
html, body, #root {
height: 100%;
margin: 0;
padding: 0;
}
body {
font-family: 'Inter', system-ui, sans-serif;
background-color: var(--color-bg-primary);
color: var(--color-text-primary);
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
/* Scrollbar styling */
::-webkit-scrollbar {
width: 6px;
height: 6px;
}
::-webkit-scrollbar-track {
background: #0f172a;
}
::-webkit-scrollbar-thumb {
background: #334155;
border-radius: 3px;
}
::-webkit-scrollbar-thumb:hover {
background: #475569;
}
/* Custom utilities */
@layer utilities {
.text-gradient {
@apply bg-gradient-to-r from-accent-cyan to-accent-emerald bg-clip-text text-transparent;
}
.border-glow {
box-shadow: 0 0 10px rgba(34, 211, 238, 0.2);
}
.card {
@apply bg-slate-900 border border-slate-800 rounded-lg;
}
.card-hover {
@apply hover:border-slate-700 hover:bg-slate-800/50 transition-all duration-200;
}
.input-field {
@apply bg-slate-900 border border-slate-700 rounded px-3 py-2 text-sm text-slate-100
placeholder:text-slate-500 focus:outline-none focus:border-accent-cyan focus:ring-1 focus:ring-accent-cyan/30;
}
.btn-primary {
@apply bg-accent-cyan text-slate-950 px-4 py-2 rounded font-medium text-sm
hover:bg-accent-cyan/90 active:bg-accent-cyan/80 transition-colors
disabled:opacity-50 disabled:cursor-not-allowed;
}
.btn-secondary {
@apply bg-slate-800 text-slate-200 px-4 py-2 rounded font-medium text-sm border border-slate-700
hover:bg-slate-700 hover:border-slate-600 active:bg-slate-600 transition-colors
disabled:opacity-50 disabled:cursor-not-allowed;
}
.btn-danger {
@apply bg-accent-rose/10 text-accent-rose px-4 py-2 rounded font-medium text-sm border border-accent-rose/30
hover:bg-accent-rose/20 active:bg-accent-rose/30 transition-colors
disabled:opacity-50 disabled:cursor-not-allowed;
}
.status-indicator {
@apply w-2 h-2 rounded-full;
}
.status-active {
@apply bg-accent-emerald animate-pulse;
}
.status-warning {
@apply bg-accent-amber;
}
.status-error {
@apply bg-accent-rose;
}
.status-inactive {
@apply bg-slate-600;
}
}
/* Monospace text */
.font-mono {
font-family: 'JetBrains Mono', 'Fira Code', monospace;
}
/* Animation for loading states */
@keyframes shimmer {
0% { background-position: -200% 0; }
100% { background-position: 200% 0; }
}
.shimmer {
background: linear-gradient(90deg, #1e293b 25%, #334155 50%, #1e293b 75%);
background-size: 200% 100%;
animation: shimmer 1.5s infinite;
}

10
frontend/src/main.tsx Normal file
View File

@@ -0,0 +1,10 @@
import React from 'react'
import ReactDOM from 'react-dom/client'
import App from './App'
import './index.css'
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<App />
</React.StrictMode>,
)

View File

@@ -0,0 +1,81 @@
import { useEffect, useState } from 'react';
import { useAuthStore } from '../stores/authStore';
import { useTargetStore } from '../stores/targetStore';
import { useScraperStore } from '../stores/scraperStore';
import { TargetList } from '../components/TargetList';
import { MainDataFeed } from '../components/MainDataFeed';
import { ScraperPanel } from '../components/ScraperPanel';
import { Header } from '../components/Header';
import { AddTargetModal } from '../components/AddTargetModal';
import { AddProfileModal } from '../components/AddProfileModal';
import { AddSessionModal } from '../components/AddSessionModal';
export function Dashboard() {
const { checkAuth } = useAuthStore();
const { fetchTargets } = useTargetStore();
const { fetchJobs, fetchSessions, initSocket, disconnectSocket } = useScraperStore();
const [showAddTarget, setShowAddTarget] = useState(false);
const [showAddProfile, setShowAddProfile] = useState(false);
const [showAddSession, setShowAddSession] = useState(false);
useEffect(() => {
// Initialize
checkAuth();
fetchTargets();
fetchJobs();
fetchSessions();
initSocket();
// Polling for updates
const interval = setInterval(() => {
fetchJobs();
fetchSessions();
}, 10000);
return () => {
clearInterval(interval);
disconnectSocket();
};
}, []);
return (
<div className="h-screen flex flex-col overflow-hidden bg-slate-950">
<Header
onAddTarget={() => setShowAddTarget(true)}
onAddSession={() => setShowAddSession(true)}
/>
{/* Main 3-column layout */}
<div className="flex-1 flex overflow-hidden">
{/* Left column - Target List */}
<aside className="w-72 flex-shrink-0 border-r border-slate-800 overflow-hidden">
<TargetList onAddProfile={() => setShowAddProfile(true)} />
</aside>
{/* Center column - Main Data Feed */}
<main className="flex-1 overflow-hidden">
<MainDataFeed />
</main>
{/* Right column - Scraper Status & Logs */}
<aside className="w-80 flex-shrink-0 border-l border-slate-800 overflow-hidden">
<ScraperPanel />
</aside>
</div>
{/* Modals */}
{showAddTarget && (
<AddTargetModal onClose={() => setShowAddTarget(false)} />
)}
{showAddProfile && (
<AddProfileModal onClose={() => setShowAddProfile(false)} />
)}
{showAddSession && (
<AddSessionModal onClose={() => setShowAddSession(false)} />
)}
</div>
);
}

View File

@@ -0,0 +1,148 @@
import { useState, useEffect } from 'react';
import { useNavigate } from 'react-router-dom';
import { useAuthStore } from '../stores/authStore';
import { Shield, AlertCircle, Lock, Eye, EyeOff } from 'lucide-react';
export function LoginPage() {
const [password, setPassword] = useState('');
const [error, setError] = useState('');
const [loading, setLoading] = useState(false);
const [showPassword, setShowPassword] = useState(false);
const navigate = useNavigate();
const { login, checkAuth, isAuthenticated } = useAuthStore();
useEffect(() => {
// Check if already authenticated
checkAuth().then((authenticated) => {
if (authenticated) {
navigate('/');
}
});
}, []);
useEffect(() => {
if (isAuthenticated) {
navigate('/');
}
}, [isAuthenticated, navigate]);
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
setError('');
setLoading(true);
try {
await login(password);
navigate('/');
} catch (err: any) {
setError(err.message || 'Authentication failed');
} finally {
setLoading(false);
}
};
return (
<div className="min-h-screen flex items-center justify-center bg-slate-950 px-4">
{/* Background pattern */}
<div className="absolute inset-0 overflow-hidden">
<div className="absolute inset-0 bg-gradient-to-br from-slate-950 via-slate-900 to-slate-950" />
<div className="absolute inset-0 opacity-30">
<div className="absolute top-1/4 left-1/4 w-96 h-96 bg-accent-cyan/5 rounded-full blur-3xl" />
<div className="absolute bottom-1/4 right-1/4 w-96 h-96 bg-accent-violet/5 rounded-full blur-3xl" />
</div>
{/* Grid pattern */}
<svg className="absolute inset-0 w-full h-full opacity-[0.03]">
<defs>
<pattern id="grid" width="40" height="40" patternUnits="userSpaceOnUse">
<path d="M 40 0 L 0 0 0 40" fill="none" stroke="currentColor" strokeWidth="1" />
</pattern>
</defs>
<rect width="100%" height="100%" fill="url(#grid)" />
</svg>
</div>
<div className="relative w-full max-w-md">
{/* Logo/Header */}
<div className="text-center mb-8">
<div className="inline-flex items-center justify-center w-16 h-16 rounded-2xl bg-gradient-to-br from-accent-cyan/20 to-accent-violet/20 border border-accent-cyan/30 mb-4">
<Shield className="w-8 h-8 text-accent-cyan" />
</div>
<h1 className="text-2xl font-bold text-slate-100 mb-1">OSINT Platform</h1>
<p className="text-sm text-slate-500">Private Intelligence Automation</p>
</div>
{/* Login Card */}
<div className="card p-6 border-slate-800 shadow-xl shadow-black/20">
<div className="flex items-center gap-2 mb-6 pb-4 border-b border-slate-800">
<Lock className="w-4 h-4 text-slate-500" />
<span className="text-sm font-medium text-slate-400">Master Authentication</span>
</div>
<form onSubmit={handleSubmit} className="space-y-4">
{error && (
<div className="flex items-center gap-2 p-3 rounded bg-accent-rose/10 border border-accent-rose/20 text-accent-rose text-sm">
<AlertCircle className="w-4 h-4 flex-shrink-0" />
<span>{error}</span>
</div>
)}
<div>
<label htmlFor="password" className="block text-sm font-medium text-slate-400 mb-2">
Master Password
</label>
<div className="relative">
<input
id="password"
type={showPassword ? 'text' : 'password'}
value={password}
onChange={(e) => setPassword(e.target.value)}
className="input-field w-full pr-10 font-mono"
placeholder="Enter your master password"
autoFocus
disabled={loading}
/>
<button
type="button"
onClick={() => setShowPassword(!showPassword)}
className="absolute right-3 top-1/2 -translate-y-1/2 text-slate-500 hover:text-slate-300 transition-colors"
>
{showPassword ? <EyeOff className="w-4 h-4" /> : <Eye className="w-4 h-4" />}
</button>
</div>
</div>
<button
type="submit"
disabled={!password || loading}
className="btn-primary w-full flex items-center justify-center gap-2"
>
{loading ? (
<>
<div className="w-4 h-4 border-2 border-slate-950 border-t-transparent rounded-full animate-spin" />
<span>Authenticating...</span>
</>
) : (
<>
<Shield className="w-4 h-4" />
<span>Authenticate</span>
</>
)}
</button>
</form>
<div className="mt-6 pt-4 border-t border-slate-800">
<div className="flex items-center gap-2 text-xs text-slate-600">
<div className="w-1.5 h-1.5 rounded-full bg-accent-emerald animate-pulse" />
<span>Encrypted connection Rate limited</span>
</div>
</div>
</div>
{/* Footer */}
<p className="text-center text-xs text-slate-600 mt-6">
All access attempts are logged and monitored.
</p>
</div>
</div>
);
}

View File

@@ -0,0 +1,86 @@
import { create } from 'zustand';
import { persist } from 'zustand/middleware';
interface AuthState {
isAuthenticated: boolean;
token: string | null;
login: (password: string) => Promise<boolean>;
logout: () => void;
checkAuth: () => Promise<boolean>;
}
export const useAuthStore = create<AuthState>()(
persist(
(set, get) => ({
isAuthenticated: false,
token: null,
login: async (password: string) => {
try {
const response = await fetch('/api/auth/login', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ password }),
credentials: 'include',
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Login failed');
}
const data = await response.json();
set({ isAuthenticated: true, token: data.token });
return true;
} catch (error) {
set({ isAuthenticated: false, token: null });
throw error;
}
},
logout: async () => {
try {
await fetch('/api/auth/logout', {
method: 'POST',
credentials: 'include',
});
} catch (e) {
// Ignore errors
}
set({ isAuthenticated: false, token: null });
},
checkAuth: async () => {
const { token } = get();
if (!token) {
set({ isAuthenticated: false });
return false;
}
try {
const response = await fetch('/api/auth/verify', {
headers: {
Authorization: `Bearer ${token}`,
},
credentials: 'include',
});
if (response.ok) {
set({ isAuthenticated: true });
return true;
}
} catch (e) {
// Token invalid
}
set({ isAuthenticated: false, token: null });
return false;
},
}),
{
name: 'osint-auth',
partialize: (state) => ({ token: state.token }),
}
)
);

View File

@@ -0,0 +1,206 @@
import { create } from 'zustand';
import { io, Socket } from 'socket.io-client';
import { useAuthStore } from './authStore';
export interface ScraperJob {
id: string;
target_id?: string;
profile_id?: string;
platform: string;
status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
progress: number;
result?: string;
error?: string;
target_name?: string;
profile_username?: string;
started_at?: string;
completed_at?: string;
created_at: string;
logs?: ScraperLog[];
}
export interface ScraperLog {
id: number;
job_id: string;
level: string;
message: string;
timestamp: string;
}
export interface Session {
id: string;
platform: string;
session_name: string;
user_agent?: string;
proxy?: string;
status: 'active' | 'expired' | 'invalid';
last_validated?: string;
created_at: string;
updated_at: string;
}
interface ScraperState {
jobs: ScraperJob[];
activeJobs: ScraperJob[];
sessions: Session[];
logs: Map<string, ScraperLog[]>;
socket: Socket | null;
loading: boolean;
error: string | null;
initSocket: () => void;
disconnectSocket: () => void;
fetchJobs: () => Promise<void>;
fetchSessions: () => Promise<void>;
startJob: (config: { platform: string; profileUrl?: string; targetId?: string; profileId?: string }) => Promise<ScraperJob>;
cancelJob: (jobId: string) => Promise<void>;
subscribeToJob: (jobId: string) => void;
addSession: (data: { platform: string; session_name: string; cookies: any[]; user_agent?: string }) => Promise<void>;
deleteSession: (id: string) => Promise<void>;
}
export const useScraperStore = create<ScraperState>((set, get) => ({
jobs: [],
activeJobs: [],
sessions: [],
logs: new Map(),
socket: null,
loading: false,
error: null,
initSocket: () => {
const token = useAuthStore.getState().token;
if (!token) return;
const socket = io({
auth: { token },
});
socket.on('connect', () => {
console.log('Socket connected');
});
socket.on('scraper:log', (log: ScraperLog & { jobId: string }) => {
const logs = new Map(get().logs);
const jobLogs = logs.get(log.jobId) || [];
logs.set(log.jobId, [...jobLogs, log]);
set({ logs });
});
socket.on('scraper:status', (update: { jobId: string; status: string; progress?: number }) => {
const jobs = get().jobs.map(job =>
job.id === update.jobId
? { ...job, status: update.status as any, progress: update.progress ?? job.progress }
: job
);
set({ jobs });
});
socket.on('scraper:jobUpdate', () => {
get().fetchJobs();
});
set({ socket });
},
disconnectSocket: () => {
const { socket } = get();
if (socket) {
socket.disconnect();
set({ socket: null });
}
},
fetchJobs: async () => {
set({ loading: true });
try {
const response = await fetch('/api/scraper/jobs', { credentials: 'include' });
if (!response.ok) throw new Error('Failed to fetch jobs');
const jobs = await response.json();
const statusResponse = await fetch('/api/scraper/status', { credentials: 'include' });
const statusData = await statusResponse.json();
set({
jobs,
activeJobs: statusData.activeJobs || [],
loading: false
});
} catch (error: any) {
set({ error: error.message, loading: false });
}
},
fetchSessions: async () => {
try {
const response = await fetch('/api/sessions', { credentials: 'include' });
if (!response.ok) throw new Error('Failed to fetch sessions');
const sessions = await response.json();
set({ sessions });
} catch (error: any) {
set({ error: error.message });
}
},
startJob: async (config) => {
const response = await fetch('/api/scraper/start', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
platform: config.platform,
profile_url: config.profileUrl,
target_id: config.targetId,
profile_id: config.profileId,
}),
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to start job');
const job = await response.json();
// Subscribe to job updates
get().subscribeToJob(job.id);
await get().fetchJobs();
return job;
},
cancelJob: async (jobId: string) => {
const response = await fetch(`/api/scraper/jobs/${jobId}/cancel`, {
method: 'POST',
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to cancel job');
await get().fetchJobs();
},
subscribeToJob: (jobId: string) => {
const { socket } = get();
if (socket) {
socket.emit('subscribe:scraper', jobId);
}
},
addSession: async (data) => {
const response = await fetch('/api/sessions', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(data),
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to add session');
await get().fetchSessions();
},
deleteSession: async (id: string) => {
const response = await fetch(`/api/sessions/${id}`, {
method: 'DELETE',
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to delete session');
await get().fetchSessions();
},
}));

View File

@@ -0,0 +1,143 @@
import { create } from 'zustand';
export interface Target {
id: string;
name: string;
notes?: string;
profile_count?: number;
created_at: string;
updated_at: string;
profiles?: SocialProfile[];
}
export interface SocialProfile {
id: string;
target_id: string;
platform: string;
username?: string;
profile_url?: string;
profile_data?: string;
last_scraped?: string;
created_at: string;
}
interface TargetState {
targets: Target[];
selectedTarget: Target | null;
loading: boolean;
error: string | null;
fetchTargets: () => Promise<void>;
fetchTarget: (id: string) => Promise<void>;
selectTarget: (target: Target | null) => void;
createTarget: (name: string, notes?: string) => Promise<Target>;
updateTarget: (id: string, name: string, notes?: string) => Promise<void>;
deleteTarget: (id: string) => Promise<void>;
addProfile: (targetId: string, platform: string, username?: string, profileUrl?: string) => Promise<void>;
deleteProfile: (targetId: string, profileId: string) => Promise<void>;
}
export const useTargetStore = create<TargetState>((set, get) => ({
targets: [],
selectedTarget: null,
loading: false,
error: null,
fetchTargets: async () => {
set({ loading: true, error: null });
try {
const response = await fetch('/api/targets', { credentials: 'include' });
if (!response.ok) throw new Error('Failed to fetch targets');
const data = await response.json();
const targets = data.map((t: any) => ({
...t,
updated_at: t.updatedAt || t.updated_at,
created_at: t.createdAt || t.created_at
}));
set({ targets, loading: false });
} catch (error: any) {
set({ error: error.message, loading: false });
}
},
fetchTarget: async (id: string) => {
set({ loading: true, error: null });
try {
const response = await fetch(`/api/targets/${id}`, { credentials: 'include' });
if (!response.ok) throw new Error('Failed to fetch target');
const t = await response.json();
const target = {
...t,
updated_at: t.updatedAt || t.updated_at,
created_at: t.createdAt || t.created_at
};
set({ selectedTarget: target, loading: false });
} catch (error: any) {
set({ error: error.message, loading: false });
}
},
selectTarget: (target) => {
set({ selectedTarget: target });
},
createTarget: async (name: string, notes?: string) => {
const response = await fetch('/api/targets', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name, notes }),
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to create target');
const target = await response.json();
await get().fetchTargets();
return target;
},
updateTarget: async (id: string, name: string, notes?: string) => {
const response = await fetch(`/api/targets/${id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name, notes }),
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to update target');
await get().fetchTargets();
if (get().selectedTarget?.id === id) {
await get().fetchTarget(id);
}
},
deleteTarget: async (id: string) => {
const response = await fetch(`/api/targets/${id}`, {
method: 'DELETE',
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to delete target');
if (get().selectedTarget?.id === id) {
set({ selectedTarget: null });
}
await get().fetchTargets();
},
addProfile: async (targetId: string, platform: string, username?: string, profileUrl?: string) => {
const response = await fetch(`/api/targets/${targetId}/profiles`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ platform, username, profile_url: profileUrl }),
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to add profile');
await get().fetchTarget(targetId);
await get().fetchTargets();
},
deleteProfile: async (targetId: string, profileId: string) => {
const response = await fetch(`/api/targets/${targetId}/profiles/${profileId}`, {
method: 'DELETE',
credentials: 'include',
});
if (!response.ok) throw new Error('Failed to delete profile');
await get().fetchTarget(targetId);
await get().fetchTargets();
},
}));

View File

@@ -0,0 +1,48 @@
/** @type {import('tailwindcss').Config} */
export default {
content: [
"./index.html",
"./src/**/*.{js,ts,jsx,tsx}",
],
theme: {
extend: {
colors: {
// Midnight Utility Theme
slate: {
950: '#020617',
900: '#0f172a',
800: '#1e293b',
700: '#334155',
600: '#475569',
500: '#64748b',
400: '#94a3b8',
300: '#cbd5e1',
200: '#e2e8f0',
100: '#f1f5f9',
},
accent: {
cyan: '#22d3ee',
emerald: '#10b981',
amber: '#f59e0b',
rose: '#f43f5e',
violet: '#8b5cf6',
},
},
fontFamily: {
sans: ['Inter', 'system-ui', 'sans-serif'],
mono: ['JetBrains Mono', 'Fira Code', 'monospace'],
},
animation: {
'pulse-slow': 'pulse 3s cubic-bezier(0.4, 0, 0.6, 1) infinite',
'glow': 'glow 2s ease-in-out infinite alternate',
},
keyframes: {
glow: {
'0%': { boxShadow: '0 0 5px rgba(34, 211, 238, 0.3)' },
'100%': { boxShadow: '0 0 20px rgba(34, 211, 238, 0.6)' },
},
},
},
},
plugins: [],
}

25
frontend/tsconfig.json Normal file
View File

@@ -0,0 +1,25 @@
{
"compilerOptions": {
"target": "ES2020",
"useDefineForClassFields": true,
"lib": ["ES2020", "DOM", "DOM.Iterable"],
"module": "ESNext",
"skipLibCheck": true,
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"baseUrl": ".",
"paths": {
"@/*": ["src/*"]
}
},
"include": ["src"],
"references": [{ "path": "./tsconfig.node.json" }]
}

View File

@@ -0,0 +1,11 @@
{
"compilerOptions": {
"composite": true,
"skipLibCheck": true,
"module": "ESNext",
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"strict": true
},
"include": ["vite.config.ts"]
}

25
frontend/vite.config.ts Normal file
View File

@@ -0,0 +1,25 @@
import { defineConfig } from 'vite'
import react from '@vitejs/plugin-react'
// https://vitejs.dev/config/
export default defineConfig({
plugins: [react()],
server: {
port: 5173,
proxy: {
'/api': {
target: 'http://localhost:3001',
changeOrigin: true,
},
'/socket.io': {
target: 'http://localhost:3001',
changeOrigin: true,
ws: true,
},
},
},
build: {
outDir: 'dist',
sourcemap: false,
},
})

372
package-lock.json generated Normal file
View File

@@ -0,0 +1,372 @@
{
"name": "osint-platform",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "osint-platform",
"version": "1.0.0",
"devDependencies": {
"concurrently": "^8.2.2"
}
},
"node_modules/@babel/runtime": {
"version": "7.28.6",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz",
"integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/ansi-regex": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dev": true,
"license": "MIT",
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dev": true,
"license": "MIT",
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/chalk/node_modules/supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"license": "MIT",
"dependencies": {
"has-flag": "^4.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/cliui": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
"dev": true,
"license": "ISC",
"dependencies": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.1",
"wrap-ansi": "^7.0.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true,
"license": "MIT"
},
"node_modules/concurrently": {
"version": "8.2.2",
"resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.2.tgz",
"integrity": "sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==",
"dev": true,
"license": "MIT",
"dependencies": {
"chalk": "^4.1.2",
"date-fns": "^2.30.0",
"lodash": "^4.17.21",
"rxjs": "^7.8.1",
"shell-quote": "^1.8.1",
"spawn-command": "0.0.2",
"supports-color": "^8.1.1",
"tree-kill": "^1.2.2",
"yargs": "^17.7.2"
},
"bin": {
"conc": "dist/bin/concurrently.js",
"concurrently": "dist/bin/concurrently.js"
},
"engines": {
"node": "^14.13.0 || >=16.0.0"
},
"funding": {
"url": "https://github.com/open-cli-tools/concurrently?sponsor=1"
}
},
"node_modules/date-fns": {
"version": "2.30.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
"integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.21.0"
},
"engines": {
"node": ">=0.11"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/date-fns"
}
},
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
"dev": true,
"license": "MIT"
},
"node_modules/escalade": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
"integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"dev": true,
"license": "ISC",
"engines": {
"node": "6.* || 8.* || >= 10.*"
}
},
"node_modules/has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/lodash": {
"version": "4.17.23",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz",
"integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==",
"dev": true,
"license": "MIT"
},
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/rxjs": {
"version": "7.8.2",
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.1.0"
}
},
"node_modules/shell-quote": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz",
"integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/spawn-command": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz",
"integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==",
"dev": true
},
"node_modules/string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"dev": true,
"license": "MIT",
"dependencies": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/strip-ansi": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
"dev": true,
"license": "MIT",
"dependencies": {
"ansi-regex": "^5.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/supports-color": {
"version": "8.1.1",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
"integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"has-flag": "^4.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/supports-color?sponsor=1"
}
},
"node_modules/tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
"integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
"dev": true,
"license": "MIT",
"bin": {
"tree-kill": "cli.js"
}
},
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"dev": true,
"license": "0BSD"
},
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
"strip-ansi": "^6.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
"dev": true,
"license": "ISC",
"engines": {
"node": ">=10"
}
},
"node_modules/yargs": {
"version": "17.7.2",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
"integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
"dev": true,
"license": "MIT",
"dependencies": {
"cliui": "^8.0.1",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"string-width": "^4.2.3",
"y18n": "^5.0.5",
"yargs-parser": "^21.1.1"
},
"engines": {
"node": ">=12"
}
},
"node_modules/yargs-parser": {
"version": "21.1.1",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
"dev": true,
"license": "ISC",
"engines": {
"node": ">=12"
}
}
}
}

19
package.json Normal file
View File

@@ -0,0 +1,19 @@
{
"name": "osint-platform",
"version": "1.0.0",
"description": "Private OSINT Automation Platform",
"private": true,
"scripts": {
"dev": "concurrently \"npm run dev:backend\" \"npm run dev:frontend\"",
"dev:backend": "cd backend && npm run dev",
"dev:frontend": "cd frontend && npm run dev",
"build": "npm run build:frontend && npm run build:backend",
"build:frontend": "cd frontend && npm run build",
"build:backend": "cd backend && npm run build",
"start": "cd backend && npm start",
"install:all": "npm install && cd backend && npm install && cd ../frontend && npm install"
},
"devDependencies": {
"concurrently": "^8.2.2"
}
}