00001 <?php 00009 function convertLinks() { 00010 global $wgDBtype; 00011 if( $wgDBtype == 'postgres' ) { 00012 wfOut( "Links table already ok on Postgres.\n" ); 00013 return; 00014 } 00015 00016 wfOut( "Converting links table to ID-ID...\n" ); 00017 00018 global $wgLang, $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname; 00019 global $noKeys, $logPerformance, $fh; 00020 00021 $tuplesAdded = $numBadLinks = $curRowsRead = 0; #counters etc 00022 $totalTuplesInserted = 0; # total tuples INSERTed into links_temp 00023 00024 $reportCurReadProgress = true; #whether or not to give progress reports while reading IDs from cur table 00025 $curReadReportInterval = 1000; #number of rows between progress reports 00026 00027 $reportLinksConvProgress = true; #whether or not to give progress reports during conversion 00028 $linksConvInsertInterval = 1000; #number of rows per INSERT 00029 00030 $initialRowOffset = 0; 00031 #$finalRowOffset = 0; # not used yet; highest row number from links table to process 00032 00033 # Overwrite the old links table with the new one. If this is set to false, 00034 # the new table will be left at links_temp. 00035 $overwriteLinksTable = true; 00036 00037 # Don't create keys, and so allow duplicates in the new links table. 00038 # This gives a huge speed improvement for very large links tables which are MyISAM. (What about InnoDB?) 00039 $noKeys = false; 00040 00041 00042 $logPerformance = false; # output performance data to a file 00043 $perfLogFilename = "convLinksPerf.txt"; 00044 #-------------------------------------------------------------------- 00045 00046 $dbw = wfGetDB( DB_MASTER ); 00047 list ($cur, $links, $links_temp, $links_backup) = $dbw->tableNamesN( 'cur', 'links', 'links_temp', 'links_backup' ); 00048 00049 $res = $dbw->query( "SELECT l_from FROM $links LIMIT 1" ); 00050 if ( $dbw->fieldType( $res, 0 ) == "int" ) { 00051 wfOut( "Schema already converted\n" ); 00052 return; 00053 } 00054 00055 $res = $dbw->query( "SELECT COUNT(*) AS count FROM $links" ); 00056 $row = $dbw->fetchObject($res); 00057 $numRows = $row->count; 00058 $dbw->freeResult( $res ); 00059 00060 if ( $numRows == 0 ) { 00061 wfOut( "Updating schema (no rows to convert)...\n" ); 00062 createTempTable(); 00063 } else { 00064 if ( $logPerformance ) { $fh = fopen ( $perfLogFilename, "w" ); } 00065 $baseTime = $startTime = getMicroTime(); 00066 # Create a title -> cur_id map 00067 wfOut( "Loading IDs from $cur table...\n" ); 00068 performanceLog ( "Reading $numRows rows from cur table...\n" ); 00069 performanceLog ( "rows read vs seconds elapsed:\n" ); 00070 00071 $dbw->bufferResults( false ); 00072 $res = $dbw->query( "SELECT cur_namespace,cur_title,cur_id FROM $cur" ); 00073 $ids = array(); 00074 00075 while ( $row = $dbw->fetchObject( $res ) ) { 00076 $title = $row->cur_title; 00077 if ( $row->cur_namespace ) { 00078 $title = $wgLang->getNsText( $row->cur_namespace ) . ":$title"; 00079 } 00080 $ids[$title] = $row->cur_id; 00081 $curRowsRead++; 00082 if ($reportCurReadProgress) { 00083 if (($curRowsRead % $curReadReportInterval) == 0) { 00084 performanceLog( $curRowsRead . " " . (getMicroTime() - $baseTime) . "\n" ); 00085 wfOut( "\t$curRowsRead rows of $cur table read.\n" ); 00086 } 00087 } 00088 } 00089 $dbw->freeResult( $res ); 00090 $dbw->bufferResults( true ); 00091 wfOut( "Finished loading IDs.\n\n" ); 00092 performanceLog( "Took " . (getMicroTime() - $baseTime) . " seconds to load IDs.\n\n" ); 00093 #-------------------------------------------------------------------- 00094 00095 # Now, step through the links table (in chunks of $linksConvInsertInterval rows), 00096 # convert, and write to the new table. 00097 createTempTable(); 00098 performanceLog( "Resetting timer.\n\n" ); 00099 $baseTime = getMicroTime(); 00100 wfOut( "Processing $numRows rows from $links table...\n" ); 00101 performanceLog( "Processing $numRows rows from $links table...\n" ); 00102 performanceLog( "rows inserted vs seconds elapsed:\n" ); 00103 00104 for ($rowOffset = $initialRowOffset; $rowOffset < $numRows; $rowOffset += $linksConvInsertInterval) { 00105 $sqlRead = "SELECT * FROM $links "; 00106 $sqlRead = $dbw->limitResult($sqlRead, $linksConvInsertInterval,$rowOffset); 00107 $res = $dbw->query($sqlRead); 00108 if ( $noKeys ) { 00109 $sqlWrite = array("INSERT INTO $links_temp (l_from,l_to) VALUES "); 00110 } else { 00111 $sqlWrite = array("INSERT IGNORE INTO $links_temp (l_from,l_to) VALUES "); 00112 } 00113 00114 $tuplesAdded = 0; # no tuples added to INSERT yet 00115 while ( $row = $dbw->fetchObject($res) ) { 00116 $fromTitle = $row->l_from; 00117 if ( array_key_exists( $fromTitle, $ids ) ) { # valid title 00118 $from = $ids[$fromTitle]; 00119 $to = $row->l_to; 00120 if ( $tuplesAdded != 0 ) { 00121 $sqlWrite[] = ","; 00122 } 00123 $sqlWrite[] = "($from,$to)"; 00124 $tuplesAdded++; 00125 } else { # invalid title 00126 $numBadLinks++; 00127 } 00128 } 00129 $dbw->freeResult($res); 00130 #wfOut( "rowOffset: $rowOffset\ttuplesAdded: $tuplesAdded\tnumBadLinks: $numBadLinks\n" ); 00131 if ( $tuplesAdded != 0 ) { 00132 if ($reportLinksConvProgress) { 00133 wfOut( "Inserting $tuplesAdded tuples into $links_temp..." ); 00134 } 00135 $dbw->query( implode("",$sqlWrite) ); 00136 $totalTuplesInserted += $tuplesAdded; 00137 if ($reportLinksConvProgress) 00138 wfOut( " done. Total $totalTuplesInserted tuples inserted.\n" ); 00139 performanceLog( $totalTuplesInserted . " " . (getMicroTime() - $baseTime) . "\n" ); 00140 } 00141 } 00142 wfOut( "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n\n" ); 00143 performanceLog( "$totalTuplesInserted valid titles and $numBadLinks invalid titles were processed.\n" ); 00144 performanceLog( "Total execution time: " . (getMicroTime() - $startTime) . " seconds.\n" ); 00145 if ( $logPerformance ) { fclose ( $fh ); } 00146 } 00147 #-------------------------------------------------------------------- 00148 00149 if ( $overwriteLinksTable ) { 00150 $dbConn = Database::newFromParams( $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname ); 00151 if (!($dbConn->isOpen())) { 00152 wfOut( "Opening connection to database failed.\n" ); 00153 return; 00154 } 00155 # Check for existing links_backup, and delete it if it exists. 00156 wfOut( "Dropping backup links table if it exists..." ); 00157 $dbConn->query( "DROP TABLE IF EXISTS $links_backup", DB_MASTER); 00158 wfOut( " done.\n" ); 00159 00160 # Swap in the new table, and move old links table to links_backup 00161 wfOut( "Swapping tables '$links' to '$links_backup'; '$links_temp' to '$links'..." ); 00162 $dbConn->query( "RENAME TABLE links TO $links_backup, $links_temp TO $links", DB_MASTER ); 00163 wfOut( " done.\n\n" ); 00164 00165 $dbConn->close(); 00166 wfOut( "Conversion complete. The old table remains at $links_backup;\n" ); 00167 wfOut( "delete at your leisure.\n" ); 00168 } else { 00169 wfOut( "Conversion complete. The converted table is at $links_temp;\n" ); 00170 wfOut( "the original links table is unchanged.\n" ); 00171 } 00172 } 00173 00174 #-------------------------------------------------------------------- 00175 00176 function createTempTable() { 00177 global $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname; 00178 global $noKeys; 00179 $dbConn = Database::newFromParams( $wgDBserver, $wgDBadminuser, $wgDBadminpassword, $wgDBname ); 00180 00181 if (!($dbConn->isOpen())) { 00182 wfOut( "Opening connection to database failed.\n" ); 00183 return; 00184 } 00185 $links_temp = $dbConn->tableName( 'links_temp' ); 00186 00187 wfOut( "Dropping temporary links table if it exists..." ); 00188 $dbConn->query( "DROP TABLE IF EXISTS $links_temp"); 00189 wfOut( " done.\n" ); 00190 00191 wfOut( "Creating temporary links table..." ); 00192 if ( $noKeys ) { 00193 $dbConn->query( "CREATE TABLE $links_temp ( " . 00194 "l_from int(8) unsigned NOT NULL default '0', " . 00195 "l_to int(8) unsigned NOT NULL default '0')"); 00196 } else { 00197 $dbConn->query( "CREATE TABLE $links_temp ( " . 00198 "l_from int(8) unsigned NOT NULL default '0', " . 00199 "l_to int(8) unsigned NOT NULL default '0', " . 00200 "UNIQUE KEY l_from(l_from,l_to), " . 00201 "KEY (l_to))"); 00202 } 00203 wfOut( " done.\n\n" ); 00204 } 00205 00206 function performanceLog( $text ) { 00207 global $logPerformance, $fh; 00208 if ( $logPerformance ) { 00209 fwrite( $fh, $text ); 00210 } 00211 } 00212 00213 function getMicroTime() { # return time in seconds, with microsecond accuracy 00214 list($usec, $sec) = explode(" ", microtime()); 00215 return ((float)$usec + (float)$sec); 00216 }