first commit

This commit is contained in:
2024-05-16 16:05:41 +02:00
commit adf8283ae0
197 changed files with 26666 additions and 0 deletions
+42
View File
@@ -0,0 +1,42 @@
## Backend
# Dev
npm run dev
http://localhost:3000
## Docker Database Dev
```bash
docker-compose -f docker-compose-dev.yml build
docker-compose -f docker-compose-dev.yml up
```
## Agenda
http://localhost:3000/dash/
## API
# Lot
GET http://localhost:3000/api/lot/getInfos/https%3A%2F%2Fwww.interencheres.com%2Fvehicules%2Fvehicules-624955%2Flot-75622389.html
GET http://localhost:3000/api/lot/getPictures/https%3A%2F%2Fwww.interencheres.com%2Fvehicules%2Fvehicules-624955%2Flot-75622389.html
POST http://localhost:3000/api/lot/NextItem
POST http://localhost:3000/api/lot/AuctionedItem
POST http://localhost:3000/api/lot/Bid
# Sale
GET http://localhost:3000/api/sale/getSaleInfos/https%3A%2F%2Fwww.interencheres.com%2Fvehicules%2Fvehicules-624955
GET http://localhost:3000/api/sale/followSale/624955
GET http://localhost:3000/api/sale/sale/624955
POST http://localhost:3000/api/sale/sale
PUT http://localhost:3000/api/sale/sale/624955
DELETE http://localhost:3000/api/sale/sale/624955
GET http://localhost:3000/api/sale/getAll
GET http://localhost:3000/api/sale/getByUrl/https%3A%2F%2Fwww.interencheres.com%2Fvehicules%2Fvehicules-624955
# Favorite
POST http://localhost:3000/api/favorite/save
GET http://localhost:3000/api/favorite/getAll
# Prod
npm run start
+27
View File
@@ -0,0 +1,27 @@
const asyncHandler = require("express-async-handler");
const { save, getAll } = require("../services/favorites");
exports.save = asyncHandler(async (req, res, next) => {
try{
let result = await save(req.body);
console.log(result);
res.status(204).send();
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.getAll = asyncHandler(async (req, res, next) => {
console.log("controller getAll");
try{
let result = await getAll();
console.log(result);
res.status(200).send(result);
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
+167
View File
@@ -0,0 +1,167 @@
const asyncHandler = require("express-async-handler");
const fetch = require('node-fetch');
const { LotDb } = require("../services/lotDb");
const lotDb = new LotDb();
const { SaleDb } = require("../services/saleDb");
const saleDb = new SaleDb();
const ApiURL = "http://host.docker.internal:3020/api";
// scrapping
exports.getInfos = asyncHandler(async (req, res, next) => {
let url = req.params.url
url = encodeURIComponent(url);
fetch(ApiURL+'/lot/getInfos/'+url)
.then(response => response.json())
.then(data => {
res.json(data);
})
.catch(error => {
console.error(error);
});
});
exports.getPictures = asyncHandler(async (req, res, next) => {
let url = req.params.url
url = encodeURIComponent(url);
fetch(ApiURL+'/lot/getPictures/'+url)
.then(response => response.json())
.then(data => {
res.json(data);
})
.catch(error => {
console.error(error);
});
});
exports.getLotsBySale = asyncHandler(async (req, res, next) => {
let id = req.params.id
const Sale = await saleDb.get(id);
if(!Sale){
console.error("Sale not found");
return res.status(404).send("Sale not found");
}
Lots = await lotDb.getBySaleId(Sale._id.toString(),Sale.platform);
res.json(Lots);
});
// Follow Sale
exports.NextItem = asyncHandler(async (req, res, next) => {
try{
Sale = await saleDb.getByIDPlatform(req.body.idSalePlatform, req.body.platform);
if(!Sale){
console.error("Sale not found");
return res.status(404).send("Sale not found");
}
let Lot = await lotDb.getByIDPlatform(req.body.idPlatform, req.body.platform);
if(Lot == null){
console.log("Creating new Lot");
Lot = {
idPlatform: String(req.body.idPlatform),
platform: req.body.platform,
timestamp: req.body.timestamp,
lotNumber: String(req.body.lotNumber),
title: req.body.title,
description: req.body.description,
EstimateLow: req.body.EstimateLow,
EstimateHigh: req.body.EstimateHigh,
RawData: req.body.RawData,
sale_id: Sale._id,
}
await lotDb.post(Lot);
}else{
console.log("Updating Lot");
Lot.timestamp = req.body.timestamp;
Lot.lotNumber = String(req.body.lotNumber);
Lot.title = req.body.title;
Lot.description = req.body.description;
Lot.EstimateLow = req.body.EstimateLow;
Lot.EstimateHigh = req.body.EstimateHigh;
Lot.RawData = req.body.RawData;
await lotDb.put(Lot._id, Lot);
}
res.status(204).send();
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.Bid = asyncHandler(async (req, res, next) => {
try{
let Lot = await lotDb.getByIDPlatform(req.body.idPlatform, req.body.platform);
if(Lot){
console.log("Update Lot Bid");
BidInfo = {
timestamp: req.body.timestamp,
amount: req.body.amount,
auctioned_type: req.body.auctioned_type,
}
// If Lot.BidInfo doesn't exist, initialize it as an empty array
if (!Lot.Bids) {
Lot.Bids = [];
}
// Add BidInfo to the array
Lot.Bids.push(BidInfo);
await lotDb.put(Lot._id, Lot);
}else{
console.error("Lot not found");
return res.status(404).send("Lot not found");
}
res.status(204).send();
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.AuctionedItem = asyncHandler(async (req, res, next) => {
try{
let Lot = await lotDb.getByIDPlatform(req.body.idPlatform, req.body.platform);
if(Lot){
console.log("Update Lot AuctionedItem");
Lot.auctioned = {
timestamp: req.body.timestamp,
amount: req.body.amount,
auctioned_type: req.body.auctioned_type,
sold: req.body.sold,
}
await lotDb.put(Lot._id, Lot);
}else{
console.error("Lot not found");
return res.status(404).send("Lot not found");
}
res.status(204).send();
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
+282
View File
@@ -0,0 +1,282 @@
const asyncHandler = require("express-async-handler");
const moment = require('moment-timezone');
const { ObjectId } = require('mongodb');
const { SaleDb } = require("../services/saleDb");
const saleDb = new SaleDb();
const { LotDb } = require("../services/lotDb");
const lotDb = new LotDb();
const agenda = require('../services/agenda');
const {Agent} = require('../services/agent');
const agent = new Agent();
exports.getSaleInfos = asyncHandler(async (req, res, next) => {
let url = req.params.url
agent.getSaleInfos(url)
.then(data => {
return res.status(200).json(data);
})
.catch(error => {
console.error(error);
return res.status(500).send(error);
});
// url = encodeURIComponent(url);
// fetch(ApiAgentURL+'/sale/getSaleInfos/'+url)
// .then(response => response.json())
// .then(data => {
// res.json(data);
// })
// .catch(error => {
// console.error(error);
// });
});
exports.prepareSale = asyncHandler(async (req, res, next) => {
try{
const id = req.params.id;
agent.prepareSale(id)
.then(data => {
return res.status(200).json({"message": "Lots created"});
})
.catch(error => {
console.error(error);
return res.status(500).send(error);
});
// url = encodeURIComponent(url);
// fetch(ApiAgentURL+'/sale/getLotList/'+url)
// .then(response => response.json())
// .then(async data => {
// console.log(data);
// for (let lot of data) {
// lot.sale_id = Sale._id
// await lotDb.post(lot);
// }
// res.status(200).send({"message": "Lots created"})
// })
// .catch(error => {
// console.error(error);
// return res.status(500).send(error);
// });
}catch(err){
console.error(err);
return res.status(500).send(err);
}
});
exports.followSale = asyncHandler(async (req, res, next) => {
try{
const id = req.params.id;
agent.followSale(id)
.then(data => {
res.status(200).send(data);
})
.catch(error => {
console.error(error);
return res.status(500).send(error);
});
}catch(err){
console.error(err);
return res.status(500).send(err);
}
});
// DB
exports.get = asyncHandler(async (req, res, next) => {
try{
const id = req.params.id;
let result = await saleDb.get(id);
res.status(200).send(result);
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.post = asyncHandler(async (req, res, next) => {
try{
// check if double
let Sale = await saleDb.getByIDPlatform(req.body.idPlatform, req.body.platform);
if(Sale){
return res.status(500).send("Sale already exists");
}
let createData = await saleDb.post(req.body);
console.log(createData.insertedId);
const NowParis = moment.tz(new Date(),"Europe/Paris")
// Scheduling the Prepare job
const dateSaleMinus24Hours = moment.tz(req.body.date, "Europe/Paris").subtract(24, 'hours');
if(dateSaleMinus24Hours.isAfter(NowParis)){
const jobPrepare = agenda.create('prepareSale', { saleId: createData.insertedId });
jobPrepare.schedule(dateSaleMinus24Hours.toDate());
await jobPrepare.save();
}else{ console.log("Sale is less than 24 hours away, no Prepare Job");}
// Scheduling the Follow job
const dateSale = moment.tz(req.body.date, "Europe/Paris");
if(dateSale.isAfter(NowParis)){
const jobFollow = agenda.create('followSale', { saleId: createData.insertedId });
jobFollow.schedule(dateSale.toDate());
await jobFollow.save();
}else{ console.log("Sale is in the past, no Follow Job");}
res.status(204).send();
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.put = asyncHandler(async (req, res, next) => {
try{
const id = req.params.id;
let updatedDocument = { ...req.body };
delete updatedDocument._id;
console.log(updatedDocument);
let result = await saleDb.put(id, updatedDocument);
console.log(result);
res.status(200).send(result);
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.delete = asyncHandler(async (req, res, next) => {
try{
const id = req.params.id;
// Remove all lots linked to the sale
console.log("Deleting lots sale_id: "+id);
await lotDb.deleteAllLotBySaleId(id);
// Remove the sale
await saleDb.remove(id);
//remove the Jobs
const JobSale = await agenda.jobs({ 'data.saleId': new ObjectId(id) });
for (const job of JobSale) {
await job.remove();
}
res.status(200).send({"message": "Sale and Lots deleted"});
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
// Fucntions
exports.getAll = asyncHandler(async (req, res, next) => {
try{
let result = await saleDb.getAll();
res.status(200).send(result);
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.getByUrl = asyncHandler(async (req, res, next) => {
try{
let url = req.params.url
url = decodeURIComponent(url);
let result = await saleDb.getByUrl(url);
//console.log(result);
res.status(200).send(result);
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
exports.postProcessing = asyncHandler(async (req, res, next) => {
try{
const id = req.params.id;
Sale = await saleDb.get(id);
if(!Sale){
console.error("Sale not found");
return res.status(404).send("Sale not found");
}
Lots = await lotDb.getBySaleId(Sale._id.toString(),Sale.platform);
let startTime = 0;
if (Array.isArray(Lots[0].Bids)) {
startTime = Lots[0].Bids[0].timestamp;
}else{
startTime = Lots[0].timestamp;
}
let LastBid = [...Lots].reverse().find(lot => lot.auctioned !== undefined);
let endTime = 0;
if (Array.isArray(LastBid.Bids)) {
endTime = LastBid.Bids[LastBid.Bids.length-1].timestamp;
}else{
endTime = LastBid.timestamp;
}
console.log("Start Time: "+startTime);
console.log("End Time: "+endTime);
let duration = endTime-startTime;
let totalAmount = 0;
for (let lot of Lots) {
if (lot.auctioned) {
totalAmount += lot.auctioned.amount;
}
}
function calculateMedian(array) {
array.sort((a, b) => a - b);
let middleIndex = Math.floor(array.length / 2);
if (array.length % 2 === 0) { // array has an even length
return (array[middleIndex - 1] + array[middleIndex]) / 2;
} else { // array has an odd length
return array[middleIndex];
}
}
const amounts = Lots.map(lot => lot.auctioned?.amount).filter(Boolean);
//console.error(Lots);
let postProcessing = {
nbrLots: Lots.length,
duration: duration,
durationPerLots: (duration/Lots.length).toFixed(0),
totalAmount: totalAmount,
averageAmount: (totalAmount/Lots.length).toFixed(2),
medianAmount: calculateMedian(amounts).toFixed(2),
}
console.log(postProcessing);
Sale.postProcessing = postProcessing;
await saleDb.put(Sale._id, Sale);
res.status(200).send({"message": "Post Processing done"});
}catch(err){
console.log(err);
return res.status(500).send(err);
}
});
+27
View File
@@ -0,0 +1,27 @@
const express = require('express')
const app = express()
var bodyParser = require('body-parser');
app.use(bodyParser.json())
const cors = require('cors');
app.use(cors());
// Enable preflight requests for all routes
app.options('*', cors());
// Agenda Scheduller
const agenda = require('./services/agenda');
(async function() {
await agenda.start();
})();
// Agenda UI
var Agendash = require("agendash");
app.use("/dash", Agendash(agenda));
// routes
app.use('/api/lot', require('./routes/lot'));
app.use('/api/sale', require('./routes/sale'));
app.use('/api/favorite', require('./routes/favorite'));
module.exports = app
+10
View File
@@ -0,0 +1,10 @@
{
"type": "node",
"request": "attach",
"name": "Attach to Process",
"restart": true,
"address": "127.0.0.1",
"port": 53481,
"localRoot": "${workspaceFolder}",
"remoteRoot": "${workspaceFolder}"
}
+5416
View File
File diff suppressed because it is too large Load Diff
+27
View File
@@ -0,0 +1,27 @@
{
"name": "jucundus",
"version": "1.0.0",
"description": "",
"main": "server.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node server.js",
"dev": "nodemon --watch ./ server.js --ignore node_modules/"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@angular/cli": "^17.1.3",
"@hokify/agenda": "^6.3.0",
"agendash": "^4.0.0",
"cors": "^2.8.5",
"express": "^4.18.2",
"express-async-handler": "^1.2.0",
"mongodb": "^6.5.0",
"node-fetch": "^2.7.0"
},
"devDependencies": {
"nodemon": "^3.0.3"
}
}
+7
View File
@@ -0,0 +1,7 @@
const controllers = require('../controllers/favorite')
const router = require('express').Router()
router.post('/save/', controllers.save)
router.get('/getAll/', controllers.getAll)
module.exports = router
+12
View File
@@ -0,0 +1,12 @@
const controllers = require('../controllers/lot')
const router = require('express').Router()
router.get('/getInfos/:url', controllers.getInfos)
router.get('/getPictures/:url', controllers.getPictures)
router.get('/getLotsBySale/:id', controllers.getLotsBySale)
router.post('/NextItem/', controllers.NextItem)
router.post('/AuctionedItem/', controllers.AuctionedItem)
router.post('/Bid/', controllers.Bid)
module.exports = router
+21
View File
@@ -0,0 +1,21 @@
const controllers = require('../controllers/sale')
const router = require('express').Router()
// AuctionAgent
router.get('/getSaleInfos/:url', controllers.getSaleInfos)
router.get('/prepareSale/:id', controllers.prepareSale)
router.get('/followSale/:id', controllers.followSale)
// DB
router.get('/sale/:id', controllers.get)
router.post('/sale/', controllers.post)
router.put('/sale/:id', controllers.put)
router.delete('/sale/:id', controllers.delete)
router.get('/getAll/', controllers.getAll)
router.get('/getByUrl/:url', controllers.getByUrl)
router.get('/postProcessing/:id', controllers.postProcessing)
module.exports = router
+5
View File
@@ -0,0 +1,5 @@
const app = require('./index.js')
const port = process.env.PORT || '3000'
app.listen(port, '0.0.0.0', () => {
console.log('Server listening on port '+port);
});
+23
View File
@@ -0,0 +1,23 @@
const Agenda = require("agenda");
const agenda = new Agenda({ db: { address: "mongodb://db:27017/agendaDb" } });
const {Agent} = require('./agent');
const agent = new Agent();
// Define a job
agenda.define('followSale', async (job, done) => {
const { saleId } = job.attrs.data;
agent.followSale(saleId)
.then(data => {;
done();
})
});
agenda.define('prepareSale', async (job, done) => {
const { saleId } = job.attrs.data;
agent.prepareSale(saleId)
.then(data => {;
done();
})
});
module.exports = agenda;
+89
View File
@@ -0,0 +1,89 @@
const fetch = require('node-fetch');
const { LotDb } = require("./lotDb");
const lotDb = new LotDb();
const { SaleDb } = require("./saleDb");
const saleDb = new SaleDb();
const moment = require('moment-timezone');
const Agent = class
{
constructor()
{
this.ApiAgentURL = "http://host.docker.internal:3020/api";
}
async getSaleInfos(url)
{
return new Promise((resolve, reject) => {
url = encodeURIComponent(url);
fetch(this.ApiAgentURL+'/sale/getSaleInfos/'+url)
.then(response => response.json())
.then(data => {
resolve(data);
})
.catch(error => {
reject(error);
});
});
}
async prepareSale(id)
{
return new Promise(async (resolve, reject) => {
let Sale = await saleDb.get(id);
const DateSale = moment.tz(Sale.date, "Europe/Paris");
const NowParis = moment.tz(new Date(),"Europe/Paris")
if (NowParis.isBefore(DateSale)){
let url = Sale.url
url = encodeURIComponent(url);
fetch(this.ApiAgentURL+'/sale/getLotList/'+url)
.then(response => response.json())
.then( async data => {
for (let lot of data) {
lot.sale_id = Sale._id
await lotDb.post(lot);
}
resolve(data);
})
.catch(error => {
reject(error);
});
}else{
console.log("Sale started or finished");
resolve([]);
}
});
}
async followSale(id)
{
return new Promise(async (resolve, reject) => {
let Sale = await saleDb.get(id);
let url = Sale.url
url = encodeURIComponent(url);
fetch(this.ApiAgentURL+'/sale/followSale/'+url)
.then(response => response.json())
.then(async data => {
// set the Sale status to following
Sale.status = "following";
Sale = await saleDb.put(id, Sale);
resolve(data);
})
.catch(error => {
reject(error);
});
});
}
}
module.exports = {Agent};
+18
View File
@@ -0,0 +1,18 @@
const MongoClient = require("mongodb").MongoClient;
const connectionString = "mongodb://db:27017";
const client = new MongoClient(connectionString);
let db;
const connectDb = async () => {
if (db) return db;
try {
const conn = await client.connect();
db = conn.db("jucundus");
return db;
} catch(e) {
console.error(e);
}
};
module.exports = connectDb;
+27
View File
@@ -0,0 +1,27 @@
const connectDb = require("./db");
const save = async (newDocument) => {
const db = await connectDb();
if (!db) {
throw new Error('Database not connected');
}
const collection = db.collection("Favorites");
let result = await collection.insertOne(newDocument);
return result;
};
const getAll = async () => {
const db = await connectDb();
if (!db) {
throw new Error('Database not connected');
}
const collection = db.collection("Favorites");
let result = await collection.find({}).toArray();
return result;
};
module.exports = { save, getAll };
+75
View File
@@ -0,0 +1,75 @@
const { ObjectId } = require('mongodb');
const connectDb = require("./db");
const LotDb = class
{
constructor()
{
this.getCollection();
}
async getCollection()
{
const db = await connectDb();
if (!db) {
throw new Error('Database not connected');
}
this.collection = db.collection("Lots");
}
// CRUD
async get(id)
{
let result = await this.collection.findOne({_id: new ObjectId(id)});
return result;
}
async post(newDocument)
{
let result = await this.collection.insertOne(newDocument);
return result;
}
async put(id, data)
{
let result = await this.collection.updateOne({_id: new ObjectId(id)}, {$set: data});
return result;
}
async remove(id)
{
let result = await this.collection.deleteOne({_id: new ObjectId(id)});
return result;
}
// Fucntions
async getAll()
{
let result = await this.collection.find({}).toArray();
return result;
}
async getBySaleId(idSalePlatform, platformName)
{
console.log(platformName);
let result = await this.collection.find({sale_id: new ObjectId(idSalePlatform), platform: platformName});
return result.toArray();
}
async getByIDPlatform(idLotPlatform, platformName)
{
let result = await this.collection.findOne({idPlatform: String(idLotPlatform), platform: platformName});
return result;
}
async deleteAllLotBySaleId(sale_id){
let result = await this.collection.deleteMany({sale_id: new ObjectId(sale_id)});
return result;
}
}
module.exports = {LotDb};
+67
View File
@@ -0,0 +1,67 @@
const { ObjectId } = require('mongodb');
const connectDb = require("./db");
const SaleDb = class
{
constructor()
{
this.getCollection();
}
async getCollection()
{
const db = await connectDb();
if (!db) {
throw new Error('Database not connected');
}
this.collection = db.collection("Sales");
}
// CRUD
async get(id)
{
let result = await this.collection.findOne({_id: new ObjectId(id)});
return result;
}
async post(newDocument)
{
let result = await this.collection.insertOne(newDocument);
return result;
}
async put(id, data)
{
let result = await this.collection.updateOne({_id: new ObjectId(id)}, {$set: data});
return result;
}
async remove(id)
{
let result = await this.collection.deleteOne({_id: new ObjectId(id)});
return result;
}
// Fucntions
async getAll()
{
let result = await this.collection.find({}).toArray();
return result;
}
async getByUrl(url)
{
let result = await this.collection.findOne({url: url});
return result;
}
async getByIDPlatform(idSalePlatform, platformName)
{
let result = await this.collection.findOne({idPlatform: String(idSalePlatform), platform: String(platformName)});
return result;
}
}
module.exports = { SaleDb };