first commit

scrapper/Dockerfile.dev

@@ -0,0 +1,41 @@
+FROM node:slim
+
+# We don't need the standalone Chromium
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+
+# Install Google Chrome Stable and fonts
+# Note: this installs the necessary libs to make the browser work with Puppeteer.
+RUN apt-get update && apt-get install gnupg wget -y && \
+  wget --quiet --output-document=- https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/apt/trusted.gpg.d/google-archive.gpg && \
+  sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' && \
+  apt-get update && \
+  apt-get install google-chrome-stable -y --no-install-recommends && \
+  rm -rf /var/lib/apt/lists/*
+
+# Setting up the work directory
+WORKDIR /scrapper
+
+#Copying all the files in our project
+COPY . .
+
+# Installing dependencies
+RUN npm install
+
+    # Add user so we don't need --no-sandbox.
+    # same layer as npm install to keep re-chowned files from using up several hundred MBs more space
+# RUN groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
+#     && mkdir -p /home/pptruser/Downloads \
+#     && chown -R pptruser:pptruser /home/pptruser \
+#     && chown -R pptruser:pptruser ./node_modules \
+#     && chown -R pptruser:pptruser ./package.json \
+#     && chown -R pptruser:pptruser ./package-lock.json
+
+# # Run everything after as non-privileged user.
+# USER pptruser
+
+# Starting our application
+#CMD [ "npm", "run", "debug-cluster" ]
+CMD [ "npm", "run", "start" ]
+
+# Exposing server port
+EXPOSE 80

scrapper/app.js

@@ -0,0 +1,44 @@
+const express = require('express')
+const app = express()
+
+var bodyParser = require('body-parser');
+app.use(bodyParser.json())
+
+//const puppeteer = require('puppeteer');
+const puppeteer = require('puppeteer-extra');
+const pluginStealth = require('puppeteer-extra-plugin-stealth');
+puppeteer.use(pluginStealth())
+
+const puppeteerCluster = require('./middleware/puppeteerCluster');
+const { Cluster } = require('puppeteer-cluster');
+
+(async () => {
+    
+
+    cluster = await Cluster.launch({
+      concurrency: Cluster.CONCURRENCY_BROWSER,
+      maxConcurrency: 6,
+      //monitor: true,
+      timeout: 20000,
+      retryLimit: 2, 
+      puppeteerOptions: {
+        executablePath: '/usr/bin/google-chrome',
+        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu'],
+        ignoreDefaultArgs: ['--disable-extensions'],
+        headless: 'new',
+        env: {
+          TZ: 'Europe/Paris'
+        }
+      },
+      puppeteer: puppeteer
+    });
+    app.use(puppeteerCluster(cluster));
+
+    // main routes    
+    app.use('/api/sale', require('./routes/sale'));
+    app.use('/api/lot', require('./routes/lot'));
+    
+  })();
+
+
+module.exports = app

scrapper/controllers/lot.js

@@ -0,0 +1,83 @@
+const asyncHandler = require("express-async-handler");
+const {ScraperTools} = require('../AuctionServices/Scraper/Scraper.js')
+const Drouot = require('../AuctionServices/Scraper/Drouot/Drouot.js')
+const Interencheres = require('../AuctionServices/Scraper/Interencheres/Interencheres.js')
+
+let getAuctionPlatform = function(Url){
+  
+	let AuctionPlatform
+	let STools = new ScraperTools();
+	
+	switch (STools.detectPlatform(Url)) {
+  
+	  case STools._CONST_INTERENCHERES:
+		AuctionPlatform = new Interencheres(Url);
+		break;
+	  
+	  case STools._CONST_DROUOT:
+		AuctionPlatform = new Drouot(Url);		
+		break;
+  
+	  default:
+		break;
+	}
+  
+	return AuctionPlatform
+}
+
+let CleanUrl = function(url){
+  
+	if(String(url).split("http").length > 1){
+		url = 'http'+String(url).split("http")[1]
+	}else{
+		url = ""
+	}
+
+	return url
+}
+
+// Display list of all pictures of a lot.
+exports.getPictures = asyncHandler(async (req, res, next) => {
+	let url = req.params.url	
+	url = decodeURIComponent(url);
+	
+	url = CleanUrl(url)
+	if(url == ""){
+		res.status(400).send("URL not supported")
+	}
+	
+	try{
+		let AuctionPlatform = getAuctionPlatform(url);
+		if(AuctionPlatform){
+			const PictList = await req.puppeteerCluster.execute(AuctionPlatform.getPictures);
+			res.json(PictList);
+		}else{
+			res.status(400).send("URL not supported")
+		}
+	}catch(e){
+		res.status(500).send("Error: "+e)
+	}		
+	
+});
+
+exports.getInfos = asyncHandler(async (req, res, next) => {
+	let url = req.params.url	
+	url = decodeURIComponent(url);
+	
+	url = CleanUrl(url)
+	if(url == ""){
+		res.status(400).send("URL not supported")
+	}
+	
+	try{
+		let AuctionPlatform = getAuctionPlatform(url);
+		if(AuctionPlatform){
+			const LotInfos = await req.puppeteerCluster.execute(AuctionPlatform.getLotInfos);
+			res.json(LotInfos);
+		}else{
+			res.status(400).send("URL not supported")
+		}
+	}catch(e){
+		res.status(500).send("Error: "+e)
+	}		
+});

scrapper/controllers/sale.js

@@ -0,0 +1,126 @@
+const asyncHandler = require("express-async-handler");
+const fetch = require('node-fetch');
+const {ScraperTools} = require('../AuctionServices/Scraper/Scraper.js')
+
+const Drouot = require('../AuctionServices/Scraper/Drouot/Drouot.js')
+const Interencheres = require('../AuctionServices/Scraper/Interencheres/Interencheres.js')
+
+
+let getAuctionPlatform = function(Url){
+  
+	let AuctionPlatform
+	let STools = new ScraperTools();
+  
+	switch (STools.detectPlatform(Url)) {
+  
+	  case STools._CONST_INTERENCHERES:
+		AuctionPlatform = new Interencheres(Url);
+		break;
+	  
+	  case STools._CONST_DROUOT:
+		AuctionPlatform = new Drouot(Url);
+		break;
+  
+	  default:
+		break;
+	}
+  
+	return AuctionPlatform
+}
+
+let CleanUrl = function(url){
+  
+	if(String(url).split("http").length > 1){
+		url = 'http'+String(url).split("http")[1]
+	}else{
+		url = ""
+	}
+
+	return url
+}
+
+// ## PUPPETEER CLUSTER
+// get Sale info
+exports.getSaleInfos = asyncHandler(async (req, res, next) => {
+
+	let url = req.params.url	
+	url = decodeURIComponent(url);
+	
+	url = CleanUrl(url)
+	if(url == ""){
+		res.status(400).send("URL not supported")
+	}
+	
+	try{
+		let AuctionPlatform = getAuctionPlatform(url);
+		if(AuctionPlatform){
+			const SaleInfos = await req.puppeteerCluster.execute(AuctionPlatform.getSaleInfos);
+			res.json(SaleInfos);
+		}else{
+			res.status(400).send("URL not supported")
+		}
+	}catch(e){
+		res.status(500).send("Error: "+e)
+	}		
+});
+
+// get Sale Lot list
+exports.getLotList = asyncHandler(async (req, res, next) => {
+
+	let url = req.params.url	
+	url = decodeURIComponent(url);
+	
+	url = CleanUrl(url)
+	if(url == ""){
+		res.status(400).send("URL not supported")
+	}
+	
+	try{
+		let AuctionPlatform = getAuctionPlatform(url);
+		if(AuctionPlatform){
+			const LotList = await req.puppeteerCluster.execute(AuctionPlatform.getLotList);
+			res.json(LotList);
+		}else{
+			res.status(400).send("URL not supported")
+		}
+	}catch(e){
+		res.status(500).send("Error: "+e)
+	}		
+});
+
+// ## AGENT PUPPETEER
+//Follow a live Sale
+exports.followSale = asyncHandler(async (req, res, next) => {
+
+	let url = req.params.url	
+	url = decodeURIComponent(url);
+	
+	url = CleanUrl(url)
+	if(url == ""){
+		res.status(400).send("URL not supported")
+	}
+	
+	try{
+		let AuctionPlatform = getAuctionPlatform(url);
+		if(AuctionPlatform){
+			console.log('Scrapper followSale : '+encodeURIComponent(url))
+			fetch('http://agent/internApi/follow/sale/'+encodeURIComponent(url))
+    		   .then(response => {
+				console.log("fetch OK")
+				//response.json()
+				} )
+    		   .then(saleInfo => {})
+			   .catch(error => {
+				     console.error(error);
+				     throw new Error('Error: '+error)
+				});
+			
+			//res.status(200).send({status: "Following"})
+			res.status(500).send({"Error": "ok"})
+		}else{
+			res.status(400).send("URL not supported")
+		}
+	}catch(e){
+		res.status(500).send("Error: "+e)
+	}		
+});

scrapper/middleware/puppeteerCluster.js

@@ -0,0 +1,6 @@
+module.exports = (cluster) => {
+  return (req, res, next) => {    
+    req.puppeteerCluster = cluster;
+    next();
+  }
+}

scrapper/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "auctionagent",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "nodemon server.js",
+    "debug-cluster": "DEBUG='puppeteer-cluster:*' node server.js"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "body-parser": "^1.20.2",
+    "express": "^4.18.2",
+    "express-async-handler": "^1.2.0",
+    "moment-timezone": "^0.5.45",
+    "node-fetch": "^2.6.1",
+    "puppeteer": "^21.10.0",
+    "puppeteer-cluster": "^0.23.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  },
+  "devDependencies": {
+    "nodemon": "^3.0.3"
+  }
+}

scrapper/routes/lot.js

@@ -0,0 +1,7 @@
+const controllers = require('../controllers/lot')
+const router = require('express').Router()
+
+router.get('/getPictures/:url', controllers.getPictures)
+router.get('/getInfos/:url', controllers.getInfos)
+
+module.exports = router

scrapper/routes/sale.js

@@ -0,0 +1,8 @@
+const controllers = require('../controllers/sale')
+const router = require('express').Router()
+
+router.get('/getSaleInfos/:url', controllers.getSaleInfos)
+router.get('/getLotList/:url', controllers.getLotList)
+router.get('/followSale/:url', controllers.followSale)
+
+module.exports = router

scrapper/server.js

@@ -0,0 +1,16 @@
+const app = require('./app.js')
+
+const port = process.env.PORT || '3020'
+app.listen(port, () => { 
+	console.log('Server listening on port '+port); 
+});
+
+process.on('unhandledRejection', (reason, promise) => {
+	console.log('Unhandled Rejection at:', promise, 'reason:', reason);
+	// Application specific logging, throwing an error, or other logic here
+  });
+  
+  process.on('uncaughtException', (err, origin) => {
+	console.log('Caught exception: ', err, 'Exception origin: ', origin);
+	// Application specific logging, throwing an error, or other logic here
+  });

scrapper/services/jucundus.js

@@ -0,0 +1,20 @@
+const fetch = require('node-fetch');
+
+class Jucundus {
+    constructor() {
+        this.url = 'http://localhost:3000/api';
+    }
+
+    setSaleStatus(status) {
+        fetch(ApiURL+'/sale/setSaleStatus/'+status)
+        .then(response => response.json())
+        .then(data => {
+            res.json(data); 
+        })
+        .catch(error => {
+            console.error(error);
+        });
+    }
+}
+
+module.exports = Jucundus