cyril/Agent
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
cyril
2024-05-16 16:11:40 +02:00
commit fe9a8e2b20
32 changed files with 8427 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AuctionServices/Scraper/Interencheres/Interencheres.js
+273
View File
@@ -0,0 +1,273 @@
// Interencheres.js
'use strict';
const {Scraper} = require('../Scraper');
const InterencheresData = require('./InterencheresData');
class Interencheres extends Scraper {
constructor(Url) {
super(Url);
this.platformData = new InterencheresData();
this.platformData.getUrlInfo(Url).then((data) => {
if(data.lotID == 0 && data.saleID == 0){
throw new Error('Invalid URL');
}
});
this._Name = 'interencheres'
this._PAGE_MAIN = ""
this._PAGE_LOGIN = ""
this._USER = ""
this._PWD = ""
this._PATH_SESSION_FILE = ".session/session_inter.json"
}
getPictures = async ({ page, data}) => {
const PictList = await this.platformData.getPictures(page, this.Url);
console.log('PictList : '+PictList)
return PictList
}
getLotInfos = async ({ page, data}) => {
console.log("getLotInfos "+this._Name+": "+this.Url)
// Navigate the page to a URL
await page.goto(this.Url);
let idLot = await this.platformData.getLotID(this.Url);
console.log('idLot : '+idLot)
// ## LotNumber
let lotNumber = await this.platformData.getLotNumber(page);
console.log('lotNumber : '+lotNumber)
// ## Estimate
let {EstimateLow, EstimateHigh} = await this.platformData.getEstimate(page);
console.log('EstimateLow : '+EstimateLow)
// ## Description
let Description = await this.platformData.getDescription(page);
//console.log('Description : '+Description)
// ## Fees
let {feesText, fees} = await this.platformData.getFees(page);
console.log('feesText : '+feesText)
console.log('fees : '+fees)
// ################
// ### SALE
let {id_sale, urlSale} = await this.platformData.getSaleIdUrl(this.Url);
console.log('SellNumber : '+id_sale)
console.log('url : '+urlSale)
let LotInfos = {
idPlatform: idLot,
platform : this._Name,
url: this.Url,
title: 'Lot '+lotNumber,
lotNumber: lotNumber,
EstimateLow: EstimateLow,
EstimateHigh: EstimateHigh,
Description: Description,
feesText: feesText,
fees: fees,
saleInfo: {
idSale: id_sale,
url: urlSale
}
}
console.log('LotInfos : '+LotInfos)
return LotInfos
}
getSaleInfos = async ({ page, data}) => {
console.log("getSaleInfos "+this._Name+": "+this.Url)
// Navigate the page to a URL
await page.goto(this.Url);
let {id_sale, urlSale} = await this.platformData.getSaleIdUrl(this.Url);
console.log('url : '+urlSale)
// ## Title
let title = await this.platformData.getSaleTitle(page);
// ## Date
let date = await this.platformData.getSaleDate(page);
// ## Location
let location = await this.platformData.getSaleLocation(page);
// ## SaleHouseName
let saleHouseName = await this.platformData.getSaleHouseName(page);
// ## Status
// ready : ready to be followed
// following : sale is followed by the AuctionAgent
// askStop : sale is followed by the AuctionAgent and the user ask to stop following
// pause : the Sale is stopped by the Auction House and ready to restart
// end : the Sale is ended
let status = 'ready'
let SaleInfo = {
idPlatform: id_sale,
platform : this._Name,
url: urlSale,
title: title,
date: date,
location: location,
saleHouseName: saleHouseName,
status: status
}
//console.log('SaleInfo : ', JSON.stringify(SaleInfo, null, 2));
return SaleInfo
}
getLotList = async ({ page, data}) => {
console.log("getLotList "+this._Name+": "+this.Url)
// Navigate the page to a URL
await page.goto(this.Url);
const LotList = await this.platformData.getLotList(page);
console.log('LotList : '+LotList)
return LotList
}
Live = async (browser) => {
console.log("Live "+this._Name+": "+this.Url)
const page = await browser.newPage();
let CheckAskStop = null;
let Socket = null;
const StopLive = async (params) => {
clearInterval(CheckAskStop);
Socket.off('Network.webSocketFrameReceived', listener);
page.close()
}
const listener = async (params) => {
let payload = params.response.payloadData
if(payload.length>1 && payload.substring(0, 2) == '42'){
payload = JSON.parse(payload.substring(2))
//console.log(payload)
const type = payload[0];
const payloadData = payload[1];
switch (type) {
case 'startSale':
break;
case 'listAuctionedItems':
break;
case 'joinedSale':
await this.JucunduNextItem(
payloadData.sale_id,
payloadData.timestamp,
payloadData.item_id,
payloadData.order_number.primary,
payloadData.title,
payloadData.description,
payloadData.pricing.estimates.min,
payloadData.pricing.estimates.max,
payloadData
);
break;
case 'auctionedItem':
await this.JucunduAuctionedItem(
payloadData.item_id,
payloadData.timestamp,
payloadData.auctioned.amount,
payloadData.auctioned.sold,
payloadData.auctioned.type
);
break;
case 'nextItem':
await this.JucunduNextItem(
payloadData.sale_id,
payloadData.timestamp,
payloadData.item_id,
payloadData.order_number.primary,
payloadData.title,
payloadData.description,
payloadData.pricing.estimates.min,
payloadData.pricing.estimates.max,
payloadData
);
break;
case 'bid':
await this.JucundusBid(
payloadData.item_id,
payloadData.timestamp,
payloadData.amount,
payloadData.auctioned_type
);
break;
case 'pauseSale':
console.error('** Pause **');
console.log(payloadData);
// await this.JucundusEndSale()
// StopLive()
break;
case 'endSale':
// await this.JucundusEndSale()
// StopLive()
break;
default:
console.error('Unknown data type:', type);
console.log(payloadData);
}
}
};
try{
await page.goto(this.Url);
Socket = await page.target().createCDPSession();
await Socket.send('Network.enable');
await Socket.send('Page.enable');
Socket.on('Network.webSocketFrameReceived', listener);
console.log('Listener set up for Network.webSocketFrameReceived event');
// check if stop was asked
CheckAskStop = setInterval(async () => {
this.JucundusCheckStop()
.then(AskStop => {
if(AskStop){
StopLive()
}
})
}, 10000); // 10000 milliseconds = 10 seconds
}catch(e){
console.log('Error : '+e)
throw new Error('Error: '+e)
}
}
};
module.exports = Interencheres
AuctionServices/Scraper/Interencheres/InterencheresData.js
+394
View File
@@ -0,0 +1,394 @@
// interencheresData.js
const {ScraperTools} = require('../Scraper');
const urlModule = require('url');
const moment = require('moment-timezone');
const { Console } = require('console');
const { title } = require('process');
class InterencheresData extends ScraperTools {
_Name = 'interencheres'
getUrlInfo = async (url) => {
let parsedUrl = new urlModule.URL(url);
let pathParts = parsedUrl.pathname.split('/').filter(Boolean);
let typeSale = pathParts[0];
let saleID = pathParts[1].split('-')[pathParts[1].split('-').length-1];
// if lot URL
let lotID = 0;
if(pathParts.length > 2){
lotID = pathParts[2].split('-')[1].split('.')[0];
}
return {
'typeSale': typeSale,
'saleID': saleID,
'lotID': lotID
}
}
// ## Lot
getPictures = async (page, Url) => {
let PictList = []
function checkDup() {
const toFindDuplicates = array => array.filter((item, index) => array.indexOf(item) !== index)
const duplicateElements = toFindDuplicates(PictList);
// if dupplicate pictures added in the array
if (duplicateElements.length > 0) {
// remove diplucated content
PictList = PictList.filter(function (elem, pos) {
return PictList.indexOf(elem) == pos;
})
// stop the process
return false
// no dupplicated picture
} else {
// continue the process
return true
}
}
page.on('response', async response => {
const url = response.url();
if (url.match("thumbor-indbupload.interencheres.com")) {
response.buffer().then(file => {
console.log("push "+url)
PictList.push(url)
});
}
});
console.log('go to : '+Url)
await page.goto(Url);
const picturesNumberXPath = [
"//div[contains(@class, 'pswp__counter')]"
]
let picturesNumberString = await this.getTextContent(picturesNumberXPath, page, 'picturesNumberXPath')
let picturesNumber = 100;
if(picturesNumberString != ''){
picturesNumber = parseInt(picturesNumberString.split(" / ")[1])
console.log('picturesNumber : '+picturesNumber)
}
let condition = true
let idx = 0
do {
const ButtonNextXPath = [
"//button[contains(@class, 'pswp__button--arrow--right')]"
]
await this.clickLink(ButtonNextXPath, page, 'ButtonNextXPath')
await page.waitForTimeout(300);
idx++
// if number of pictures found or 20 pictures checked
if(idx+1 == picturesNumber || idx==20)condition = false
} while (condition);
checkDup()
return PictList
}
getLotNumber = async (page) => {
const lotNumberXPath = [
'/html/body/div[1]/div/div/div/main/div/div/div[2]/div/div[1]/div[2]/div[2]/div[1]',
'/html/body/div[1]/div/div/div[1]/main/div/div/div[2]/div/div[1]/div[2]/div[1]/div[1]'
]
let lotNumberString = await this.getTextContent(lotNumberXPath, page, 'lotNumberXPath')
let lotNumber = '';
if(lotNumberString != ''){
lotNumber = lotNumberString.replace('Lot ', '');
}
return lotNumber
}
getEstimate = async (page) => {
const EstimateXPath = [
'/html/body/div[1]/div/div/div/main/div/div/div[2]/div/div[1]/div[2]/div[3]/div[2]/span',
'/html/body/div[1]/div/div/div[1]/main/div/div/div[2]/div/div[1]/div[2]/div[2]/div[2]/span'
]
let EstimateString = await this.getTextContent(EstimateXPath, page, 'EstimateXPath')
console.log('EstimateString : '+EstimateString)
let EstimateLow = 0
let EstimateHigh = 0
if(EstimateString != ''){
let matches = EstimateString.match(/(\d{1,3}(?:\s\d{3})*)/g);
if (matches) {
if (matches.length >= 2) {
EstimateLow = parseInt(matches[0].replace(/\s/g, ''), 10);
EstimateHigh = parseInt(matches[1].replace(/\s/g, ''), 10);
console.log('Low:', EstimateLow);
console.log('High:', EstimateHigh);
} else
if(matches.length == 1){
EstimateLow = parseInt(matches[0].replace(/\s/g, ''), 10);
EstimateHigh = 0;
} else {
console.log('Could not extract numbers.');
}
}
}
return {EstimateLow, EstimateHigh}
}
getDescription = async (page) => {
const DescriptionXPath = [
'//div[contains(@class, "description")]',
'/html/body/div[1]/div/div/div[2]/div/div[1]/div[2]/div/div[1]/div[2]/div[4]/div',
'/html/body/div[1]/div/div/div[1]/main/div/div/div[2]/div/div[1]/div[2]/div[4]/div',
'/html/body/div[1]/div/div/div[2]/div/div[1]/div[2]/div/div[1]/div[2]/div[3]/div'
]
let Description = await this.getTextContent(DescriptionXPath, page, 'DescriptionXPath')
return Description
}
getFees = async (page) => {
let feesText = ''
let fees = 0
const ButtonFeesXPath = [
'.//a[contains(text(),"Frais de vente")]',
'/html/body/div[1]/div/div/div[2]/div/div[1]/div[2]/div/div[1]/div[2]/div[3]/div[2]/a',
'/html/body/div[1]/div/div/div[1]/main/div/div/div[2]/div/div[1]/div[2]/div[3]/div[2]/a',
'/html/body/div[1]/div/div/div[2]/div/div[1]/div[2]/div/div[1]/div[2]/div[2]/div[2]/a'
]
if(await this.clickLink(ButtonFeesXPath, page, 'ButtonFeesXPath')){
const FeesXPath = [
"//strong[contains(text(), 'Frais de vente :')]/following-sibling::span",
'/html/body/div[1]/div/div/div[3]/div/div/div[2]/div/p[1]/p[1]/span',
]
feesText = await this.getTextContent(FeesXPath, page, 'FeesXPath')
if(feesText != ''){
feesText = feesText.replace(/[\n]/g, '').replace(/\s+/g, ' ').trim();
let matches = feesText.match(/(\d+(\.\d+)?)/)
if (matches) {
fees = matches[0];
}
}
}
return {feesText, fees}
}
getLotID = async (url) => {
let UrlInfo = await this.getUrlInfo(url);
let id_lot = UrlInfo.lotID
console.log('id_lot : '+id_lot)
return id_lot
}
getSaleIdUrl = async (url) => {
let UrlInfo = await this.getUrlInfo(url);
let id_sale = UrlInfo.saleID
let urlSale = url
// remove lot information if present
if (urlSale.includes('/lot-')) {
urlSale = url.split("/lot-")[0]
}
// remove parameters
if (urlSale.includes('?')) {
urlSale = urlSale.split("?")[0];
}
console.log('getSaleIdUrl urlSale : '+urlSale)
return {id_sale, urlSale}
}
// ## Lot List
_getLotInfoList = async (page, Elements) => {
let LotList = [];
for (let element of Elements) {
let Lot = {}
try{
let LotnameXPath = [
'.//a/div/div/div[5]/div/div[1]',
'.//a/div/div/div[4]/div/div[1]',
]
let Lotname = await this.getTextContentElement(LotnameXPath, page, element, 'LotnameXPath')
// idPlatform from the url
let LotUrlXPath = [
'.//a'
]
let urlLot = await this.getAttributeElement(LotUrlXPath, page, element, 'href', 'UrlListLot')
let match = urlLot.match(/lot-(.*).html/);
let idPlatform = match[1];
Lot = {
title: Lotname,
idPlatform: idPlatform,
platform: this._Name,
lotNumber: Lotname.split('Lot ')[1]
}
}catch(e){
console.error(e)
}
//console.log(LotList)
LotList.push(Lot);
};
return LotList;
}
getLotList = async (page) => {
let LotList = []
let NextBtn = false
do {
// extract Lot List
const LotListXPath = [
'//div[contains(@class, "sale-item-wrapper")]',
]
let Elements = await page.$x(LotListXPath[0]);
if (Elements.length > 0) {
LotList = [].concat(LotList, await this._getLotInfoList(page, Elements))
}
// search for the Button Next (only if enabled)
let NextPageButtonXPath = "//button[contains(@aria-label, 'Page suivante') and not(contains(@class, 'v-pagination__navigation--disabled'))]"
let NextPageButton = await page.$x(NextPageButtonXPath);
if (NextPageButton.length > 0) {
NextBtn = true
await NextPageButton[0].evaluate(b => b.click());
await page.waitForTimeout(1000);
console.log('Next Page')
}else{
NextBtn = false
}
} while (NextBtn);
return LotList
}
// ## Sale
getSaleTitle = async (page) => {
const SaleTitleXPath = [
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[1]/div/div[1]/div[2]/h1/div/div/div/div/div',
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[1]/div/div/div/h1/div/div/div/div/div',
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[1]/div/div/div/div/h1/div/div/div/div/div'
]
let SaleTitle = await this.getTextContent(SaleTitleXPath, page, 'SaleTitleXPath')
return SaleTitle
}
getSaleDate = async (page) => {
// Test if Live Sale
let BoolLive = false;
try {
const VideoXPath = [
'//*[@id="streaming-subscriber"]',
]
let VideoExists = await this.ElementExists(VideoXPath, page, 'VideoXPath')
console.log('VideoExists : '+VideoExists)
BoolLive = VideoExists
} catch (error) {}
let SaleDate;
// if futur sale
if(!BoolLive){
await page.waitForTimeout(400);
const SaleDateXPath = [
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[1]/div/div[1]/div[2]/h2/div[2]/div/div[1]/span',
]
let SaleDateString = await this.getTextContent(SaleDateXPath, page, 'SaleDateXPath')
SaleDateString = SaleDateString.trim()
console.log('SaleDateString : '+SaleDateString)
const months = {
'janvier': '01',
'février': '02',
'mars': '03',
'avril': '04',
'mai': '05',
'juin': '06',
'juillet': '07',
'août': '08',
'septembre': '09',
'octobre': '10',
'novembre': '11',
'décembre': '12'
};
let SaleDateArr = SaleDateString.split(' ');
let day = parseInt(SaleDateArr[0].length === 1 ? '0'+SaleDateArr[0] : SaleDateArr[0]);
let month = parseInt(months[SaleDateArr[1]]);
let year = parseInt(SaleDateArr[2]);
let hour = parseInt(SaleDateArr[4].split('h')[0]);
let minute = parseInt(SaleDateArr[4].split('h')[1]);
SaleDate = moment.tz([year, month - 1, day, hour, minute], 'Europe/Paris').format();
// Live Sale
}else{
SaleDate = moment.tz('Europe/Paris').format();
}
console.log('SaleDate : '+SaleDate)
return SaleDate
}
getSaleLocation = async (page) => {
const SaleLocationXPath = [
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[1]/div/div[1]/div[2]/h2/div[2]/div/div[2]/span',
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[1]/div/div/div/h2/div[2]'
]
let SaleLocation = await this.getTextContent(SaleLocationXPath, page, 'SaleLocationXPath')
return SaleLocation.trim()
}
getSaleHouseName = async (page) => {
const SaleHouseNameXPath = [
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div[2]/div/div/div[2]/div[2]/div[2]/a',
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[2]/div[3]/div/div/div/div/div[2]/div/div/div[2]/div[2]/div[2]/a',
'/html/body/div[1]/div/div/div/main/div/div/div/div/div/div[2]/div[2]/div/div/div/div/div[2]/div/div/div/div[2]/div[2]/a',
'/html/body/div[1]/div/div/div[1]/main/div/div/div/div/div/div[2]/div[3]/div/div/div/div/div[2]/div/div/div/div[2]/div[2]/a'
]
let SaleHouseName = await this.getTextContent(SaleHouseNameXPath, page, 'SaleHouseNameXPath')
SaleHouseName = SaleHouseName.replace(/[\n]/g, '').replace(/\s+/g, ' ').trim();
return SaleHouseName.trim()
}
}
module.exports = InterencheresData