quenousimporte
/
presse
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

less scraping

This commit is contained in:
quenousimporte 2024-05-23 17:00:20 +02:00
parent ad82598214
commit a5b4b1e735
1 changed files with 83 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
mediapart.php
View File

@ -1,35 +1,5 @@
<?php <?php
/*ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);*/
function cleantext( $node )
{
$content = $node->textContent;
$content = str_replace('Partager sur Facebook', '', $content);
$content = str_replace('Partager sur X (ex Twitter)', '', $content);
$content = str_replace('Lire + tard', '', $content);
$content = str_replace('Offrir l’article', '', $content);
$content = str_replace(' PDF', '', $content);
$content = str_replace('Cette fonctionnalité est actuellement en maintenance.La génération de PDF reste disponible via la fonctionnalité «Imprimer».', '', $content);
$content = str_replace("Fermer\n", '', $content);
$content = str_replace("Recommander\n", '', $content);
$content = str_replace("Commenter\n", '', $content);
$content = str_replace(' ', '', $content);
$content = str_replace("\n", "\n\n", $content);
for ($i = 0; $i < 15; $i++)
{
$content = str_replace("\n\n\n", "\n\n", $content);
}
return $content;
}
$sessionid = 'value of cookie MPSESSID'; $sessionid = 'value of cookie MPSESSID';
$feedurl = 'https://www.mediapart.fr/articles/feed'; $feedurl = 'https://www.mediapart.fr/articles/feed';
$opts = [ $opts = [
@ -56,6 +26,10 @@ foreach ($items as $item)
if (str_starts_with($item->pubDate, date("D, j M Y"))) if (str_starts_with($item->pubDate, date("D, j M Y")))
{ {
$title = $item->title; $title = $item->title;
//$category = $item->category;
$category = $item->xpath('dc:subject')[0];
$author = $item->xpath('dc:creator')[0];
$summary = $item->description;
// Add to content // Add to content
$header .= '- ' . $title . "\n"; $header .= '- ' . $title . "\n";
@ -63,11 +37,8 @@ foreach ($items as $item)
$article = file_get_contents($item->link, false, $context); $article = file_get_contents($item->link, false, $context);
$doc = new DOMDocument(); $doc = new DOMDocument();
$doc->loadHTML($article); $doc->loadHTML($article);
$finder = new DomXPath($doc);
$category = trim($finder->query('//p[@class="news__heading__top__kicker _default"]')->item(0)->textContent); $finder = new DomXPath($doc);
$summary = trim($finder->query('//p[@class="news__heading__top__intro margin-top:500"]')->item(0)->textContent);
$author = trim($finder->query('//a[@class="focus color:text-brand heading-product:100"]')->item(0)->textContent);
// clean images // clean images
$figures = $finder->query('//figure'); $figures = $finder->query('//figure');
@ -87,47 +58,27 @@ foreach ($items as $item)
$asides = $finder->query('//h2'); $asides = $finder->query('//h2');
foreach ($asides as $aside) foreach ($asides as $aside)
{ {
$aside->textContent = '*' . $aside->textContent . '*'; $aside->textContent = "\n*" . $aside->textContent . "*\n";
} }
// Output // Output
// old: add header as one block
//$part1 = $finder->query('//div[@class="news__heading grid"]');
//$result .= cleantext($part1->item(0));
$result .= "\n------\n\n"; $result .= "\n------\n\n";
$result .= "Article : " . $item->title . "\n"; $result .= "Article : " . $item->title . "\n";
$result .= 'Catégorie : ' . str_replace("\n", '', $category) . "\n"; $result .= 'Catégorie : ' . str_replace("\n", '', $category) . "\n";
$result .= 'Auteur.ice : ' . $author . "\n"; $result .= 'Auteur.ice : ' . $author . "\n";
$result .= "Introduction : " . $summary . "\n";
$result .= "\n"; $result .= "\n";
$result .= "Introduction :\n" . $summary . "\n";
// article main text // article main text
$maintext = $finder->query('//div[@class="news__body__center"]'); $maintext = $finder->query('//div[contains(@class, "paywall-restricted-content")]');
$result .= cleantext($maintext->item(0)); $result .= $maintext->item(0)->textContent;
} }
//break;
} }
// web version:
/*
file_put_contents('mediapart.txt', $header . "\n" . $result);
echo '<html><head><title>Mediapart - ' . $date . '</title></head><body>';
echo '<div><a download href="mediapart.txt">mediapart.txt</a></div>';
echo '<div>';
echo str_replace("\n", '<br>', $header) . '<br>' . str_replace("\n", '<br>', $result);
echo '</div>';
echo '</body></html>';
*/
$filename = 'mediapart' . str_replace(' ', '', $localedate) . '.txt'; $filename = 'mediapart' . str_replace(' ', '', $localedate) . '.txt';
header('Content-Type: application/text; charset=utf-8'); header('Content-Type: application/text; charset=utf-8');
header('Content-Disposition: attachment; filename=' . $filename); header('Content-Disposition: attachment; filename=' . $filename);
echo $header . "\n" . $result; echo $header . "\n" . $result;
?> ?>