From e9eb1032aeffb2c98c9df5fa5834036e7e4c2af5 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Thu, 25 Feb 2016 00:36:18 -0800 Subject: [PATCH] Cleanup on the parsenointro Removed all of the labyrinine logic that tried to parse the HTML page and replaced it with some preprocessing that strips the page of any text or tags that we don't want to deal with. This leaves a very easy to parse set that is then filtered by a simple array of known values. With this, it can find all dumps on a page including bad and redumps. Unbeknownst to me before, if a rom is bad, it has a tag that is called "Section: Bad". Makes it easier than trying to parse the tag for "red". --- wizzardRedux/admin/parsenointro.php | 156 +++++++++++++++------------- 1 file changed, 86 insertions(+), 70 deletions(-) diff --git a/wizzardRedux/admin/parsenointro.php b/wizzardRedux/admin/parsenointro.php index 1c73d4b..24975c0 100644 --- a/wizzardRedux/admin/parsenointro.php +++ b/wizzardRedux/admin/parsenointro.php @@ -11,6 +11,22 @@ TODO: Make the else block actually traverse a system and get the information dir like scene releases for Nintendo DS, 3DS, etc. which we can get as rollbacks. */ +// Create a name to field mapping for each of the findable fields +$field_mapping = array ( + "Size:" => "size", + "CRC32:" => "crc", + "MD5:" => "md5", + "SHA-1:" => "sha1", + "Decrypted CRC32:" => "crc", + "Decrypted MD5:" => "md5", + "Decrypted SHA-1:" => "sha1", + "Directory:" => "dir", + "NFO File:" => "nfo", + "Group:" => "group", + "Released" => "date", + "Section:" => "section", +); + ini_set('max_execution_time', 6000); // Set the execution time higher because DATs can be big // auto means create the mapping @@ -40,94 +56,94 @@ else { echo "Auto-generate no-intro name to system mapping

\n"; - $gameid = 1; + $gameid = 1; $skip = 0; $errorpage = false; - while(!$errorpage) + $baddumps = array(); + while (!$errorpage) { - $query = get_data("http://datomatic.no-intro.org/index.php?page=show_record&s=1&n=".str_pad($gameid, 4, "0", STR_PAD_LEFT)); - $query = explode("\n", $query); + // Retrieve the page information + $query = get_data("http://datomatic.no-intro.org/index.php?page=show_record&s=28&n=".str_pad($gameid, 4, "0", STR_PAD_LEFT)); + + // The error page case, it means time to stop the cycle + // This could result in too many page request too... not sure though + if ($query == "" || strpos($query, "I am too busy for this!")) + { + $errorpage = true; + break; + } - $section = ""; + // Replace tabs and nbsp by blank string (this make sure that spaces in names aren't removed) + $query = str_replace(" ", "", $query); + $query = str_replace("\t", "", $query); + $query = str_replace(" ", "", $query); + + // Split the page and only take the stuff under the header + $query = explode("", $query); + $query = $query[1]; + + // Split the page and only take the stuff before the sidebar and footer + $query = explode("", $query); + $query = $query[0]; + + // Remove all tags from the page to make it easier to parse + $query = strip_tags($query); + + // Get rid of all multiple newline sets (has to be done repeatedly because of how searching works) + $query = str_replace("\r\n", "\n", $query); + for ($i = 0; $i < 10; $i++) + { + $query = str_replace("\n\n", "\n", $query); + } + $query = str_replace("\n", "
\n", $query); + + // Read the processed page into an array and get rid of the first unnecesary items + $query = explode("\n", $query); + unset($query[0]); unset($query[1]); + var_dump($query); + die(); + + $rom = array(); + $next = ""; $dump = ""; foreach ($query as $line) { - if (strpos($line, "I am too busy for this!")) - { - $errorpage = true; - break; - } + $line = strip_tags($line); - if (strpos($line, "romname_section")) + // The first line that doesn't mention a trusted dump or verificaiton is the name of the ROM + if ($line != "" && $rom["name"] == "") { - $section = "romname"; - echo "Name: "; + echo "Name: ".$line."
\n"; + $rom["name"] = $line; } - elseif ($section == "romname" && strpos($line, " ")) + // Check the key half of all of split-line fields that we know of + elseif ($next == "") { - echo str_replace(" ", "", - str_replace("
", "", trim($line)))."
"; - $section = ""; - } - elseif (strpos($line, "ROM data")) - { - $dump = "rom"; - } - elseif (strpos($line, "Size")) - { - $section = "size"; - echo "Size: "; - } - elseif ($section == "size") - { - echo str_replace("", "", - str_replace("", "", $line))."
"; - $section = ""; - } - elseif (strpos($line, "CRC32")) - { - $section = "crc32"; - echo "CRC32: "; - } - elseif ($section == "crc32") - { - echo str_replace("", "", str_replace("", "", $line))."
"; - $section = ""; - } - elseif (strpos($line, "MD5")) - { - $section = "md5"; - echo "MD5: "; - } - elseif ($section == "md5" && !strpos($line, "", "", str_replace("", "", $line))."
"; - $section = ""; - } - elseif (strpos($line, "SHA-1")) - { - $section = "sha1"; - echo "SHA-1: "; - } - elseif ($section == "sha1") - { - echo str_replace("", "", str_replace("", "", $line))."
"; - $section = ""; - } + foreach (array_keys($field_mapping) as $key) + { + if ($line == $key) + { + echo $key." "; + $next = $key; + } + } + } + // Check the value half of all split-line fields that we know of else { - //echo htmlspecialchars($line)."
"; + if (key_exists($next, $field_mapping)) + { + echo $line."
\n"; + $rom[$field_mapping[$next]] == $line; + } + $next = ""; } } + $gameid++; echo "
"; - if ($gameid > 1000) - { - die(); - } } - // No-intro pages use POST to update information. - // The values are the names, not the numericals + echo "Error page hit or ran out of numbers.
"; } //https://davidwalsh.name/curl-download