less scraping
This commit is contained in:
parent
ad82598214
commit
a5b4b1e735
215
mediapart.php
215
mediapart.php
|
|
@ -1,133 +1,84 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
/*ini_set('display_errors', 1);
|
$sessionid = 'value of cookie MPSESSID';
|
||||||
ini_set('display_startup_errors', 1);
|
$feedurl = 'https://www.mediapart.fr/articles/feed';
|
||||||
error_reporting(E_ALL);*/
|
$opts = [
|
||||||
|
'http' => [
|
||||||
function cleantext( $node )
|
'method' => "GET",
|
||||||
{
|
'header' => "Accept-language: en\nCookie: MPSESSID=" . $sessionid,
|
||||||
$content = $node->textContent;
|
]
|
||||||
$content = str_replace('Partager sur Facebook', '', $content);
|
];
|
||||||
$content = str_replace('Partager sur X (ex Twitter)', '', $content);
|
$context = stream_context_create($opts);
|
||||||
|
|
||||||
$content = str_replace('Lire + tard', '', $content);
|
// load feeds
|
||||||
$content = str_replace('Offrir l’article', '', $content);
|
$feed = file_get_contents($feedurl);
|
||||||
$content = str_replace(' PDF', '', $content);
|
$xml = new SimpleXMLElement($feed);
|
||||||
$content = str_replace('Cette fonctionnalité est actuellement en maintenance.La génération de PDF reste disponible via la fonctionnalité «Imprimer».', '', $content);
|
$items = $xml->xpath("/rss/channel/item");
|
||||||
|
|
||||||
$content = str_replace("Fermer\n", '', $content);
|
setlocale(LC_ALL, 'fr_FR.UTF8', 'fr_FR','fr','fr','fra','fr_FR@euro');
|
||||||
$content = str_replace("Recommander\n", '', $content);
|
$localedate = strftime("%A %d %B %Y");
|
||||||
$content = str_replace("Commenter\n", '', $content);
|
|
||||||
|
$header = "Mediapart - " . $localedate . "\n\nSommaire:\n";
|
||||||
|
$result = '';
|
||||||
$content = str_replace(' ', '', $content);
|
|
||||||
$content = str_replace("\n", "\n\n", $content);
|
foreach ($items as $item)
|
||||||
for ($i = 0; $i < 15; $i++)
|
{
|
||||||
{
|
if (str_starts_with($item->pubDate, date("D, j M Y")))
|
||||||
$content = str_replace("\n\n\n", "\n\n", $content);
|
{
|
||||||
}
|
$title = $item->title;
|
||||||
|
//$category = $item->category;
|
||||||
return $content;
|
$category = $item->xpath('dc:subject')[0];
|
||||||
}
|
$author = $item->xpath('dc:creator')[0];
|
||||||
|
$summary = $item->description;
|
||||||
$sessionid = 'value of cookie MPSESSID';
|
|
||||||
$feedurl = 'https://www.mediapart.fr/articles/feed';
|
// Add to content
|
||||||
$opts = [
|
$header .= '- ' . $title . "\n";
|
||||||
'http' => [
|
|
||||||
'method' => "GET",
|
$article = file_get_contents($item->link, false, $context);
|
||||||
'header' => "Accept-language: en\nCookie: MPSESSID=" . $sessionid,
|
$doc = new DOMDocument();
|
||||||
]
|
$doc->loadHTML($article);
|
||||||
];
|
|
||||||
$context = stream_context_create($opts);
|
$finder = new DomXPath($doc);
|
||||||
|
|
||||||
// load feeds
|
// clean images
|
||||||
$feed = file_get_contents($feedurl);
|
$figures = $finder->query('//figure');
|
||||||
$xml = new SimpleXMLElement($feed);
|
foreach ($figures as $figure)
|
||||||
$items = $xml->xpath("/rss/channel/item");
|
{
|
||||||
|
$figure->parentNode->removeChild($figure);
|
||||||
setlocale(LC_ALL, 'fr_FR.UTF8', 'fr_FR','fr','fr','fra','fr_FR@euro');
|
}
|
||||||
$localedate = strftime("%A %d %B %Y");
|
|
||||||
|
// clean "à lire aussi" and inline quotes
|
||||||
$header = "Mediapart - " . $localedate . "\n\nSommaire:\n";
|
$asides = $finder->query('//aside');
|
||||||
$result = '';
|
foreach ($asides as $aside)
|
||||||
|
{
|
||||||
foreach ($items as $item)
|
$aside->parentNode->removeChild($aside);
|
||||||
{
|
}
|
||||||
if (str_starts_with($item->pubDate, date("D, j M Y")))
|
|
||||||
{
|
// improve section titles
|
||||||
$title = $item->title;
|
$asides = $finder->query('//h2');
|
||||||
|
foreach ($asides as $aside)
|
||||||
// Add to content
|
{
|
||||||
$header .= '- ' . $title . "\n";
|
$aside->textContent = "\n*" . $aside->textContent . "*\n";
|
||||||
|
}
|
||||||
$article = file_get_contents($item->link, false, $context);
|
|
||||||
$doc = new DOMDocument();
|
// Output
|
||||||
$doc->loadHTML($article);
|
$result .= "\n------\n\n";
|
||||||
$finder = new DomXPath($doc);
|
$result .= "Article : " . $item->title . "\n";
|
||||||
|
$result .= 'Catégorie : ' . str_replace("\n", '', $category) . "\n";
|
||||||
$category = trim($finder->query('//p[@class="news__heading__top__kicker _default"]')->item(0)->textContent);
|
$result .= 'Auteur.ice : ' . $author . "\n";
|
||||||
$summary = trim($finder->query('//p[@class="news__heading__top__intro margin-top:500"]')->item(0)->textContent);
|
$result .= "Introduction : " . $summary . "\n";
|
||||||
$author = trim($finder->query('//a[@class="focus color:text-brand heading-product:100"]')->item(0)->textContent);
|
$result .= "\n";
|
||||||
|
|
||||||
// clean images
|
// article main text
|
||||||
$figures = $finder->query('//figure');
|
$maintext = $finder->query('//div[contains(@class, "paywall-restricted-content")]');
|
||||||
foreach ($figures as $figure)
|
$result .= $maintext->item(0)->textContent;
|
||||||
{
|
}
|
||||||
$figure->parentNode->removeChild($figure);
|
}
|
||||||
}
|
|
||||||
|
$filename = 'mediapart' . str_replace(' ', '', $localedate) . '.txt';
|
||||||
// clean "à lire aussi" and inline quotes
|
|
||||||
$asides = $finder->query('//aside');
|
header('Content-Type: application/text; charset=utf-8');
|
||||||
foreach ($asides as $aside)
|
header('Content-Disposition: attachment; filename=' . $filename);
|
||||||
{
|
echo $header . "\n" . $result;
|
||||||
$aside->parentNode->removeChild($aside);
|
|
||||||
}
|
|
||||||
|
|
||||||
// improve section titles
|
|
||||||
$asides = $finder->query('//h2');
|
|
||||||
foreach ($asides as $aside)
|
|
||||||
{
|
|
||||||
$aside->textContent = '*' . $aside->textContent . '*';
|
|
||||||
}
|
|
||||||
|
|
||||||
// Output
|
|
||||||
|
|
||||||
// old: add header as one block
|
|
||||||
//$part1 = $finder->query('//div[@class="news__heading grid"]');
|
|
||||||
//$result .= cleantext($part1->item(0));
|
|
||||||
|
|
||||||
$result .= "\n------\n\n";
|
|
||||||
$result .= "Article : " . $item->title . "\n";
|
|
||||||
$result .= 'Catégorie: ' . str_replace("\n", '', $category) . "\n";
|
|
||||||
$result .= 'Auteur.ice : ' . $author . "\n";
|
|
||||||
$result .= "\n";
|
|
||||||
$result .= "Introduction :\n" . $summary . "\n";
|
|
||||||
|
|
||||||
// article main text
|
|
||||||
$maintext = $finder->query('//div[@class="news__body__center"]');
|
|
||||||
$result .= cleantext($maintext->item(0));
|
|
||||||
}
|
|
||||||
//break;
|
|
||||||
}
|
|
||||||
|
|
||||||
// web version:
|
|
||||||
|
|
||||||
/*
|
|
||||||
file_put_contents('mediapart.txt', $header . "\n" . $result);
|
|
||||||
|
|
||||||
echo '<html><head><title>Mediapart - ' . $date . '</title></head><body>';
|
|
||||||
echo '<div><a download href="mediapart.txt">mediapart.txt</a></div>';
|
|
||||||
|
|
||||||
echo '<div>';
|
|
||||||
echo str_replace("\n", '<br>', $header) . '<br>' . str_replace("\n", '<br>', $result);
|
|
||||||
echo '</div>';
|
|
||||||
echo '</body></html>';
|
|
||||||
*/
|
|
||||||
|
|
||||||
$filename = 'mediapart' . str_replace(' ', '', $localedate) . '.txt';
|
|
||||||
header('Content-Type: application/text; charset=utf-8');
|
|
||||||
header('Content-Disposition: attachment; filename=' . $filename);
|
|
||||||
|
|
||||||
echo $header . "\n" . $result;
|
|
||||||
|
|
||||||
?>
|
?>
|
||||||
Loading…
Reference in New Issue