This commit is contained in:
echo committed 2025-05-06 10:01:03 +02:00
commit 0fa3c7ac48
140 files changed
+24289

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
export declare class ScraperModule {
}
+25
View File
@@ -0,0 +1,25 @@
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ScraperModule = void 0;
const common_1 = require("@nestjs/common");
const axios_1 = require("@nestjs/axios");
const schedule_1 = require("@nestjs/schedule");
const scraper_service_1 = require("./scraper.service");
const prisma_module_1 = require("../prisma/prisma.module");
let ScraperModule = class ScraperModule {
};
exports.ScraperModule = ScraperModule;
exports.ScraperModule = ScraperModule = __decorate([
(0, common_1.Module)({
imports: [prisma_module_1.PrismaModule, axios_1.HttpModule, schedule_1.ScheduleModule.forRoot()],
providers: [scraper_service_1.ScraperService],
exports: [scraper_service_1.ScraperService],
})
], ScraperModule);
//# sourceMappingURL=scraper.module.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"scraper.module.js","sourceRoot":"","sources":["../../src/scraper/scraper.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,yCAA2C;AAC3C,+CAAkD;AAClD,uDAAmD;AACnD,2DAAuD;AAOhD,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,sCAAa;wBAAb,aAAa;IALzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,4BAAY,EAAE,kBAAU,EAAE,yBAAc,CAAC,OAAO,EAAE,CAAC;QAC7D,SAAS,EAAE,CAAC,gCAAc,CAAC;QAC3B,OAAO,EAAE,CAAC,gCAAc,CAAC;KAC1B,CAAC;GACW,aAAa,CAAG"}
+17
View File
@@ -0,0 +1,17 @@
import { PrismaService } from '../prisma/prisma.service';
import { ConfigService } from '@nestjs/config';
export declare class ScraperService {
private prisma;
private config;
constructor(prisma: PrismaService, config: ConfigService);
private readonly logger;
private parsePrice;
private parseDate;
private parseDateRange;
private getTableSelector;
private calculateDiscountPercentage;
private parseProductRow;
scrapeAllSources(): Promise<void>;
scrapeProducts(sourceUrl: string, sourceId: number): Promise<void>;
manualScrape(sourceId: number): Promise<void>;
}
+245
View File
@@ -0,0 +1,245 @@
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var ScraperService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ScraperService = void 0;
const common_1 = require("@nestjs/common");
const prisma_service_1 = require("../prisma/prisma.service");
const schedule_1 = require("@nestjs/schedule");
const cheerio = require("cheerio");
const axios_1 = require("axios");
const config_1 = require("@nestjs/config");
let ScraperService = ScraperService_1 = class ScraperService {
prisma;
config;
constructor(prisma, config) {
this.prisma = prisma;
this.config = config;
}
logger = new common_1.Logger(ScraperService_1.name);
parsePrice(price) {
const cleanPrice = price.replace(/[^\d.]/g, '');
return cleanPrice ? parseFloat(cleanPrice) : 0;
}
parseDate(dateStr) {
try {
const [day, month, year] = dateStr.split('/').map(Number);
return new Date(year, month - 1, day);
}
catch {
return null;
}
}
parseDateRange(dateRange) {
try {
const [startStr, endStr] = dateRange.split('-').map((s) => s.trim());
return {
start: this.parseDate(startStr),
end: this.parseDate(endStr),
};
}
catch {
return { start: null, end: null };
}
}
getTableSelector(sourceId) {
switch (sourceId) {
case 2:
return 'table:eq(2)';
case 12:
return '#product-table';
default:
throw new Error(`Unsupported source ID: ${sourceId}`);
}
}
calculateDiscountPercentage(regularPrice, discountedPrice) {
if (!discountedPrice || !regularPrice)
return null;
return Math.round(((regularPrice - discountedPrice) / regularPrice) * 100);
}
parseProductRow($, rowElement, sourceId) {
const cells = $(rowElement).find('td');
const getText = (index) => {
const cell = cells.eq(index);
return cell.text().trim();
};
if (sourceId === 2) {
const promotionPeriod = getText(9);
const { start, end } = this.parseDateRange(promotionPeriod);
return {
name: getText(0),
regularPrice: this.parsePrice(getText(1)),
unitPrice: getText(2) || null,
availability: getText(3).toLowerCase() === 'да',
description: getText(4) || '',
category: 'Uncategorized',
discountedPrice: this.parsePrice(getText(6)),
discountPercentage: parseFloat(getText(7)) || null,
promotionType: getText(8) || null,
promotionStart: start,
promotionEnd: end,
};
}
else if (sourceId === 12) {
const name = getText(1);
const regularPrice = this.parsePrice(getText(3));
const description = getText(2) || '';
const discountedPrice = this.parsePrice(getText(4)) || null;
return {
name,
regularPrice,
unitPrice: null,
availability: true,
description,
category: 'Uncategorized',
discountedPrice,
discountPercentage: this.calculateDiscountPercentage(regularPrice, discountedPrice),
promotionType: null,
promotionStart: null,
promotionEnd: null,
};
}
throw new Error(`Unsupported source ID: ${sourceId}`);
}
async scrapeAllSources() {
try {
const sources = await this.prisma.source.findMany();
for (const source of sources) {
try {
await this.scrapeProducts(source.url, source.id);
this.logger.log(`Successfully scraped data from source: ${source.name}`);
}
catch (error) {
this.logger.error(`Failed to scrape source ${source.name}:`, error);
continue;
}
}
}
catch (error) {
this.logger.error('Failed to fetch sources:', error);
}
}
async scrapeProducts(sourceUrl, sourceId) {
const config = {
headers: {
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0',
},
};
try {
this.logger.log(`Fetching data from URL: ${sourceUrl}`);
const response = await axios_1.default.get(sourceUrl, config);
const $ = cheerio.load(response.data);
const productTable = $(this.getTableSelector(sourceId));
if (!productTable.length) {
throw new Error('Product table not found');
}
const rows = productTable.find('tr').slice(1);
this.logger.log(`Found ${rows.length} product rows`);
let processedProducts = 0;
for (const row of rows.toArray()) {
try {
const scrapedProduct = this.parseProductRow($, row, sourceId);
if (!scrapedProduct.name)
continue;
this.logger.log(`Processing product: ${scrapedProduct.name}`);
const product = await this.prisma.product.upsert({
where: {
name_sourceId: {
name: scrapedProduct.name,
sourceId: sourceId,
},
},
create: {
name: scrapedProduct.name,
description: scrapedProduct.description,
category: scrapedProduct.category,
availability: scrapedProduct.availability,
sourceId: sourceId,
prices: {
create: {
regularPrice: scrapedProduct.regularPrice,
discountedPrice: scrapedProduct.discountedPrice,
discountPercentage: scrapedProduct.discountPercentage,
unitPrice: scrapedProduct.unitPrice,
promotionType: scrapedProduct.promotionType,
promotionStart: scrapedProduct.promotionStart,
promotionEnd: scrapedProduct.promotionEnd,
sourceId: sourceId,
},
},
},
update: {
availability: scrapedProduct.availability,
description: scrapedProduct.description,
category: scrapedProduct.category,
prices: {
create: {
regularPrice: scrapedProduct.regularPrice,
discountedPrice: scrapedProduct.discountedPrice,
discountPercentage: scrapedProduct.discountPercentage,
unitPrice: scrapedProduct.unitPrice,
promotionType: scrapedProduct.promotionType,
promotionStart: scrapedProduct.promotionStart,
promotionEnd: scrapedProduct.promotionEnd,
sourceId: sourceId,
},
},
},
});
processedProducts++;
this.logger.log(`Successfully processed product: ${product.name}`);
}
catch (error) {
if (error instanceof Error) {
this.logger.error(`Failed to process row: ${error.message}`);
}
else {
this.logger.error('Failed to process row: Unknown error');
}
}
}
this.logger.log(`Successfully processed ${processedProducts} products`);
}
catch (error) {
if (error instanceof Error) {
this.logger.error(`Failed to scrape products from source ${sourceId}: ${error.message}`);
}
else {
this.logger.error(`Failed to scrape products from source ${sourceId}: Unknown error`);
}
throw error;
}
}
async manualScrape(sourceId) {
const source = await this.prisma.source.findUnique({
where: { id: sourceId },
});
if (!source) {
throw new Error(`Source with ID ${sourceId} not found`);
}
return this.scrapeProducts(source.url, source.id);
}
};
exports.ScraperService = ScraperService;
__decorate([
(0, schedule_1.Cron)(schedule_1.CronExpression.EVERY_HOUR),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], ScraperService.prototype, "scrapeAllSources", null);
exports.ScraperService = ScraperService = ScraperService_1 = __decorate([
(0, common_1.Injectable)(),
__metadata("design:paramtypes", [prisma_service_1.PrismaService,
config_1.ConfigService])
], ScraperService);
//# sourceMappingURL=scraper.service.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"scraper.service.js","sourceRoot":"","sources":["../../src/scraper/scraper.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,6DAAyD;AACzD,+CAAwD;AACxD,mCAAmC;AACnC,iCAAkD;AAClD,2CAA+C;AAiBxC,IAAM,cAAc,sBAApB,MAAM,cAAc;IAEf;IACA;IAFV,YACU,MAAqB,EACrB,MAAqB;QADrB,WAAM,GAAN,MAAM,CAAe;QACrB,WAAM,GAAN,MAAM,CAAe;IAC5B,CAAC;IACa,MAAM,GAAG,IAAI,eAAM,CAAC,gBAAc,CAAC,IAAI,CAAC,CAAC;IAElD,UAAU,CAAC,KAAa;QAC9B,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAChD,OAAO,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAEO,SAAS,CAAC,OAAe;QAC/B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1D,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,SAAiB;QAItC,IAAI,CAAC;YACH,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACrE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;gBAC/B,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;aAC5B,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,QAAgB;QACvC,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,CAAC;gBACJ,OAAO,aAAa,CAAC;YACvB,KAAK,EAAE;gBAEL,OAAO,gBAAgB,CAAC;YAC1B;gBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,2BAA2B,CACjC,YAAoB,EACpB,eAA8B;QAE9B,IAAI,CAAC,eAAe,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC;QACnD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,eAAe,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC;IAC7E,CAAC;IAEO,eAAe,CACrB,CAAkC,EAClC,UAA2B,EAC3B,QAAgB;QAEhB,MAAM,KAAK,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE;YACxC,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;YAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC5B,CAAC,CAAC;QAEF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YAEnB,MAAM,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;YAE5D,OAAO;gBACL,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;gBAChB,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACzC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI;gBAC7B,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI;gBAC/C,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE;gBAC7B,QAAQ,EAAE,eAAe;gBACzB,eAAe,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5C,kBAAkB,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI;gBAClD,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI;gBACjC,cAAc,EAAE,KAAK;gBACrB,YAAY,EAAE,GAAG;aAClB,CAAC;QACJ,CAAC;aAAM,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;YAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YAE5D,OAAO;gBACL,IAAI;gBACJ,YAAY;gBACZ,SAAS,EAAE,IAAI;gBACf,YAAY,EAAE,IAAI;gBAClB,WAAW;gBACX,QAAQ,EAAE,eAAe;gBACzB,eAAe;gBACf,kBAAkB,EAAE,IAAI,CAAC,2BAA2B,CAClD,YAAY,EACZ,eAAe,CAChB;gBACD,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;aACnB,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,EAAE,CAAC,CAAC;IACxD,CAAC;IAGK,AAAN,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAEpD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;oBACjD,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,0CAA0C,MAAM,CAAC,IAAI,EAAE,CACxD,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,MAAM,CAAC,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;oBACpE,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,QAAgB;QACtD,MAAM,MAAM,GAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,4EAA4E;gBAEpF,iBAAiB,EAAE,gBAAgB;gBAEnC,YAAY,EAAE,kFAAkF;aAEjG;SACF,CAAC;QAEF,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;YACxD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAS,SAAS,EAAE,MAAM,CAAC,CAAC;YAC5D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAEtC,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;YAExD,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,MAAM,eAAe,CAAC,CAAC;YAErD,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBACjC,IAAI,CAAC;oBACH,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC9D,IAAI,CAAC,cAAc,CAAC,IAAI;wBAAE,SAAS;oBAEnC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uBAAuB,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;oBAE9D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;wBAC/C,KAAK,EAAE;4BACL,aAAa,EAAE;gCACb,IAAI,EAAE,cAAc,CAAC,IAAI;gCACzB,QAAQ,EAAE,QALine truncated