registryFile = $registryFile; $this->libDir = $libDir; $this->infoPrinter = $infoPrinter ?? function ( $_ ) { }; $this->errorPrinter = $errorPrinter ?? $this->infoPrinter; $this->verbosePrinter = $verbosePrinter ?? function ( $_ ) { }; // Use a temporary directory under the destination directory instead // of wfTempDir() because PHP's rename() does not work across file // systems, and the user's /tmp and $IP may be on different filesystems. $this->tmpParentDir = "{$this->libDir}/.foreign/tmp"; $cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false; $this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache"; } /** * @param string $action * @param string $module * @return bool * @throws Exception */ public function run( $action, $module ) { $actions = [ 'update', 'verify', 'make-sri' ]; if ( !in_array( $action, $actions ) ) { $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' ); return false; } $this->action = $action; $this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) ); if ( $module === 'all' ) { $modules = $this->registry; } elseif ( isset( $this->registry[ $module ] ) ) { $modules = [ $module => $this->registry[ $module ] ]; } else { $this->error( "Unknown module name.\n\nMust be one of:\n" . wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) . '.' ); return false; } foreach ( $modules as $moduleName => $info ) { $this->verbose( "\n### {$moduleName}\n\n" ); $destDir = "{$this->libDir}/$moduleName"; if ( $this->action === 'update' ) { $this->output( "... updating '{$moduleName}'\n" ); $this->verbose( "... emptying directory for $moduleName\n" ); wfRecursiveRemoveDir( $destDir ); } elseif ( $this->action === 'verify' ) { $this->output( "... verifying '{$moduleName}'\n" ); } else { $this->output( "... checking '{$moduleName}'\n" ); } $this->verbose( "... preparing {$this->tmpParentDir}\n" ); wfRecursiveRemoveDir( $this->tmpParentDir ); if ( !wfMkdirParents( $this->tmpParentDir ) ) { throw new Exception( "Unable to create {$this->tmpParentDir}" ); } if ( !isset( $info['type'] ) ) { throw new Exception( "Module '$moduleName' must have a 'type' key." ); } switch ( $info['type'] ) { case 'tar': $this->handleTypeTar( $moduleName, $destDir, $info ); break; case 'file': $this->handleTypeFile( $moduleName, $destDir, $info ); break; case 'multi-file': $this->handleTypeMultiFile( $moduleName, $destDir, $info ); break; default: throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" ); } } $this->output( "\nDone!\n" ); $this->cleanUp(); if ( $this->hasErrors ) { // The verify mode should check all modules/files and fail after, not during. return false; } return true; } private function cacheKey( $src, $integrity ) { $key = basename( $src ) . '_' . substr( $integrity, -12 ); $key = preg_replace( '/[.\/+?=_-]+/', '_', $key ); return rtrim( $key, '_' ); } /** * @param string $key * @return string|false */ private function cacheGet( $key ) { return AtEase::quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" ); } private function cacheSet( $key, $data ) { wfMkdirParents( $this->cacheDir ); file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX ); } private function fetch( $src, $integrity ) { $key = $this->cacheKey( $src, $integrity ); $data = $this->cacheGet( $key ); if ( $data ) { return $data; } $req = MediaWikiServices::getInstance()->getHttpRequestFactory() ->create( $src, [ 'method' => 'GET', 'followRedirects' => false ], __METHOD__ ); if ( !$req->execute()->isOK() ) { throw new Exception( "Failed to download resource at {$src}" ); } if ( $req->getStatus() !== 200 ) { throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" ); } $data = $req->getContent(); $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0]; $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); if ( $integrity === $actualIntegrity ) { $this->verbose( "... passed integrity check for {$src}\n" ); $this->cacheSet( $key, $data ); } elseif ( $this->action === 'make-sri' ) { $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" ); } else { throw new Exception( "Integrity check failed for {$src}\n" . "\tExpected: {$integrity}\n" . "\tActual: {$actualIntegrity}" ); } return $data; } private function handleTypeFile( $moduleName, $destDir, array $info ) { if ( !isset( $info['src'] ) ) { throw new Exception( "Module '$moduleName' must have a 'src' key." ); } $data = $this->fetch( $info['src'], $info['integrity'] ?? null ); $dest = $info['dest'] ?? basename( $info['src'] ); $path = "$destDir/$dest"; if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { throw new Exception( "File for '$moduleName' is different." ); } if ( $this->action === 'update' ) { wfMkdirParents( $destDir ); file_put_contents( "$destDir/$dest", $data ); } } private function handleTypeMultiFile( $moduleName, $destDir, array $info ) { if ( !isset( $info['files'] ) ) { throw new Exception( "Module '$moduleName' must have a 'files' key." ); } foreach ( $info['files'] as $dest => $file ) { if ( !isset( $file['src'] ) ) { throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." ); } $data = $this->fetch( $file['src'], $file['integrity'] ?? null ); $path = "$destDir/$dest"; if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { throw new Exception( "File '$dest' for '$moduleName' is different." ); } elseif ( $this->action === 'update' ) { wfMkdirParents( $destDir ); file_put_contents( "$destDir/$dest", $data ); } } } private function handleTypeTar( $moduleName, $destDir, array $info ) { $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; if ( $info['src'] === null ) { throw new Exception( "Module '$moduleName' must have a 'src' key." ); } // Download the resource to a temporary file and open it $data = $this->fetch( $info['src'], $info['integrity' ] ); $tmpFile = "{$this->tmpParentDir}/$moduleName.tar"; $this->verbose( "... writing '$moduleName' src to $tmpFile\n" ); file_put_contents( $tmpFile, $data ); $p = new PharData( $tmpFile ); $tmpDir = "{$this->tmpParentDir}/$moduleName"; $p->extractTo( $tmpDir ); unset( $data, $p ); if ( $info['dest'] === null ) { // Default: Replace the entire directory $toCopy = [ $tmpDir => $destDir ]; } else { // Expand and normalise the 'dest' entries $toCopy = []; foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { // Use glob() to expand wildcards and check existence $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); if ( !$fromPaths ) { throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." ); } foreach ( $fromPaths as $fromPath ) { $toCopy[$fromPath] = $toSubPath === null ? "$destDir/" . basename( $fromPath ) : "$destDir/$toSubPath/" . basename( $fromPath ); } } } foreach ( $toCopy as $from => $to ) { if ( $this->action === 'verify' ) { $this->verbose( "... verifying $to\n" ); if ( is_dir( $from ) ) { $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( $from, RecursiveDirectoryIterator::SKIP_DOTS ) ); /** @var SplFileInfo $file */ foreach ( $rii as $file ) { $remote = $file->getPathname(); $local = strtr( $remote, [ $from => $to ] ); if ( sha1_file( $remote ) !== sha1_file( $local ) ) { $this->error( "File '$local' is different." ); $this->hasErrors = true; } } } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) { $this->error( "File '$to' is different." ); $this->hasErrors = true; } } elseif ( $this->action === 'update' ) { $this->verbose( "... moving $from to $to\n" ); wfMkdirParents( dirname( $to ) ); if ( !rename( $from, $to ) ) { throw new Exception( "Could not move $from to $to." ); } } } } private function verbose( $text ) { ( $this->verbosePrinter )( $text ); } private function output( $text ) { ( $this->infoPrinter )( $text ); } private function error( $text ) { ( $this->errorPrinter )( $text ); } private function cleanUp() { wfRecursiveRemoveDir( $this->tmpParentDir ); // Prune the cache of files we don't recognise. $knownKeys = []; foreach ( $this->registry as $info ) { if ( $info['type'] === 'file' || $info['type'] === 'tar' ) { $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] ); } elseif ( $info['type'] === 'multi-file' ) { foreach ( $info['files'] as $file ) { $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] ); } } } foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) { if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) { unlink( $cacheFile ); } } } /** * Basic YAML parser. * * Supports only string or object values, and 2 spaces indentation. * * @todo Just ship symfony/yaml. * @param string $input * @return array */ private function parseBasicYaml( $input ) { $lines = explode( "\n", $input ); $root = []; $stack = [ &$root ]; $prev = 0; foreach ( $lines as $i => $text ) { $line = $i + 1; $trimmed = ltrim( $text, ' ' ); if ( $trimmed === '' || $trimmed[0] === '#' ) { continue; } $indent = strlen( $text ) - strlen( $trimmed ); if ( $indent % 2 !== 0 ) { throw new Exception( __METHOD__ . ": Odd indentation on line $line." ); } $depth = $indent === 0 ? 0 : ( $indent / 2 ); if ( $depth < $prev ) { // Close previous branches we can't re-enter array_splice( $stack, $depth + 1 ); } if ( !array_key_exists( $depth, $stack ) ) { throw new Exception( __METHOD__ . ": Too much indentation on line $line." ); } if ( strpos( $trimmed, ':' ) === false ) { throw new Exception( __METHOD__ . ": Missing colon on line $line." ); } $dest =& $stack[ $depth ]; if ( $dest === null ) { // Promote from null to object $dest = []; } list( $key, $val ) = explode( ':', $trimmed, 2 ); $val = ltrim( $val, ' ' ); if ( $val !== '' ) { // Add string $dest[ $key ] = $val; } else { // Add null (may become an object later) $val = null; $stack[] = &$val; $dest[ $key ] = &$val; } $prev = $depth; unset( $dest, $val ); } return $root; } }