2017-09-13 01:31:07 +00:00
# Copyright (C) 2017 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
2019-01-10 02:20:13 +00:00
require 'digest'
2017-09-13 01:31:07 +00:00
require 'fileutils'
require 'pathname'
require 'getoptlong'
SCRIPT_NAME = File . basename ( $0 )
2017-10-18 19:14:51 +00:00
COMMENT_REGEXP = / \/ \/ /
2017-09-13 01:31:07 +00:00
2018-08-24 22:54:23 +00:00
def usage ( message )
if message
puts " Error: #{ message } "
puts
end
2017-10-18 19:14:51 +00:00
puts " usage: #{ SCRIPT_NAME } [options] <sources-list-file>... "
puts " <sources-list-file> may be separate arguments or one semicolon separated string "
2017-09-13 01:31:07 +00:00
puts " --help (-h) Print this message "
puts " --verbose (-v) Adds extra logging to stderr. "
2018-11-28 22:07:45 +00:00
puts
2017-09-26 15:34:19 +00:00
puts " Required arguments: "
puts " --source-tree-path (-s) Path to the root of the source directory. "
puts " --derived-sources-path (-d) Path to the directory where the unified source files should be placed. "
puts
puts " Optional arguments: "
2017-09-20 23:08:50 +00:00
puts " --print-bundled-sources Print bundled sources rather than generating sources "
2018-12-30 21:19:09 +00:00
puts " --print-all-sources Print all sources rather than generating sources "
2018-11-28 22:07:45 +00:00
puts " --generate-xcfilelists Generate .xcfilelist files "
puts " --input-xcfilelist-path Path of the generated input .xcfilelist file "
puts " --output-xcfilelist-path Path of the generated output .xcfilelist file "
2017-09-20 23:08:50 +00:00
puts
puts " Generation options: "
2019-01-10 02:20:13 +00:00
puts " --max-cpp-bundle-count Use global sequential numbers for cpp bundle filenames and set the limit on the number "
puts " --max-obj-c-bundle-count Use global sequential numbers for Obj-C bundle filenames and set the limit on the number "
2019-10-15 05:53:17 +00:00
puts " --dense-bundle-filter Densely bundle files matching the given path glob "
2017-09-13 01:31:07 +00:00
exit 1
end
MAX_BUNDLE_SIZE = 8
2019-10-15 05:53:17 +00:00
MAX_DENSE_BUNDLE_SIZE = 64
2017-09-13 01:31:07 +00:00
$derivedSourcesPath = nil
2017-09-26 15:34:19 +00:00
$unifiedSourceOutputPath = nil
$sourceTreePath = nil
2017-09-13 01:31:07 +00:00
$verbose = false
2017-09-20 23:08:50 +00:00
$mode = :GenerateBundles
2018-11-28 22:07:45 +00:00
$inputXCFilelistPath = nil
$outputXCFilelistPath = nil
2017-10-18 19:14:51 +00:00
$maxCppBundleCount = nil
$maxObjCBundleCount = nil
2019-10-15 05:53:17 +00:00
$denseBundleFilters = [ ]
2017-09-13 01:31:07 +00:00
2017-09-20 23:08:50 +00:00
def log ( text )
$stderr . puts text if $verbose
end
2017-09-13 01:31:07 +00:00
GetoptLong . new ( [ '--help' , '-h' , GetoptLong :: NO_ARGUMENT ] ,
[ '--verbose' , '-v' , GetoptLong :: NO_ARGUMENT ] ,
2017-09-26 15:34:19 +00:00
[ '--derived-sources-path' , '-d' , GetoptLong :: REQUIRED_ARGUMENT ] ,
[ '--source-tree-path' , '-s' , GetoptLong :: REQUIRED_ARGUMENT ] ,
2017-09-20 23:08:50 +00:00
[ '--print-bundled-sources' , GetoptLong :: NO_ARGUMENT ] ,
2018-12-30 21:19:09 +00:00
[ '--print-all-sources' , GetoptLong :: NO_ARGUMENT ] ,
2018-11-28 22:07:45 +00:00
[ '--generate-xcfilelists' , GetoptLong :: NO_ARGUMENT ] ,
[ '--input-xcfilelist-path' , GetoptLong :: REQUIRED_ARGUMENT ] ,
[ '--output-xcfilelist-path' , GetoptLong :: REQUIRED_ARGUMENT ] ,
2017-09-13 01:31:07 +00:00
[ '--max-cpp-bundle-count' , GetoptLong :: REQUIRED_ARGUMENT ] ,
2019-10-15 05:53:17 +00:00
[ '--max-obj-c-bundle-count' , GetoptLong :: REQUIRED_ARGUMENT ] ,
[ '--dense-bundle-filter' , GetoptLong :: REQUIRED_ARGUMENT ] ) . each {
2017-09-13 01:31:07 +00:00
| opt , arg |
case opt
when '--help'
2018-08-24 22:54:23 +00:00
usage ( nil )
2017-09-13 01:31:07 +00:00
when '--verbose'
$verbose = true
2017-09-20 23:08:50 +00:00
when '--derived-sources-path'
2017-09-26 15:34:19 +00:00
$derivedSourcesPath = Pathname . new ( arg )
when '--source-tree-path'
$sourceTreePath = Pathname . new ( arg )
2018-08-24 22:54:23 +00:00
usage ( " Source tree #{ arg } does not exist. " ) if ! $sourceTreePath . exist?
2017-09-26 15:34:19 +00:00
when '--print-bundled-sources'
$mode = :PrintBundledSources
2018-12-30 21:19:09 +00:00
when '--print-all-sources'
$mode = :PrintAllSources
2018-11-28 22:07:45 +00:00
when '--generate-xcfilelists'
$mode = :GenerateXCFilelists
when '--input-xcfilelist-path'
$inputXCFilelistPath = arg
when '--output-xcfilelist-path'
$outputXCFilelistPath = arg
2017-09-13 01:31:07 +00:00
when '--max-cpp-bundle-count'
2017-09-20 23:08:50 +00:00
$maxCppBundleCount = arg . to_i
2017-09-13 01:31:07 +00:00
when '--max-obj-c-bundle-count'
2017-09-20 23:08:50 +00:00
$maxObjCBundleCount = arg . to_i
2019-10-15 05:53:17 +00:00
when '--dense-bundle-filter'
$denseBundleFilters . push ( arg )
2017-09-13 01:31:07 +00:00
end
}
2018-11-28 22:07:45 +00:00
$unifiedSourceOutputPath = $derivedSourcesPath + Pathname . new ( " unified-sources " )
FileUtils . mkpath ( $unifiedSourceOutputPath ) if ! $unifiedSourceOutputPath . exist? && $mode != :GenerateXCFilelists
2018-08-24 22:54:23 +00:00
usage ( " --derived-sources-path must be specified. " ) if ! $unifiedSourceOutputPath
usage ( " --source-tree-path must be specified. " ) if ! $sourceTreePath
log ( " Putting unified sources in #{ $unifiedSourceOutputPath } " )
2017-09-26 15:34:19 +00:00
2018-08-24 22:54:23 +00:00
usage ( " At least one source list file must be specified. " ) if ARGV . length == 0
2017-10-18 19:14:51 +00:00
# Even though CMake will only pass us a single semicolon separated arguemnts, we separate all the arguments for simplicity.
sourceListFiles = ARGV . to_a . map { | sourceFileList | sourceFileList . split ( " ; " ) } . flatten
2018-08-24 22:54:23 +00:00
log ( " Source files: #{ sourceListFiles } " )
2017-09-13 01:31:07 +00:00
$generatedSources = [ ]
2018-11-28 22:07:45 +00:00
$inputSources = [ ]
$outputSources = [ ]
2017-09-13 01:31:07 +00:00
2017-10-18 19:14:51 +00:00
class SourceFile
attr_reader :unifiable , :fileIndex , :path
def initialize ( file , fileIndex )
2017-09-20 23:08:50 +00:00
@unifiable = true
2017-10-18 19:14:51 +00:00
@fileIndex = fileIndex
2017-09-20 23:08:50 +00:00
2017-10-18 19:14:51 +00:00
attributeStart = file =~ / @ /
2017-09-20 23:08:50 +00:00
if attributeStart
2017-10-18 19:14:51 +00:00
# We want to make sure we skip the first @ so split works correctly
attributesText = file [ ( attributeStart + 1 ) .. file . length ]
2017-09-20 23:08:50 +00:00
attributesText . split ( / \ s*@ / ) . each {
| attribute |
2017-10-18 19:14:51 +00:00
case attribute . strip
2017-09-20 23:08:50 +00:00
when " no-unify "
@unifiable = false
2017-10-18 19:14:51 +00:00
else
raise " unknown attribute: #{ attribute } "
2017-09-20 23:08:50 +00:00
end
}
2017-10-18 19:14:51 +00:00
file = file [ 0 .. ( attributeStart - 1 ) ]
2017-09-20 23:08:50 +00:00
end
2017-10-18 19:14:51 +00:00
@path = Pathname . new ( file . strip )
end
def <=> ( other )
return @path . dirname < = > other . path . dirname if @path . dirname != other . path . dirname
return @path . basename < = > other . path . basename if @fileIndex == other . fileIndex
@fileIndex < = > other . fileIndex
2017-09-20 23:08:50 +00:00
end
2017-09-26 15:34:19 +00:00
def derived?
return @derived if @derived != nil
2017-10-18 19:14:51 +00:00
@derived = ! ( $sourceTreePath + self . path ) . exist?
2017-09-26 15:34:19 +00:00
end
2017-10-18 19:14:51 +00:00
def to_s
2018-11-28 22:07:45 +00:00
if $mode == :GenerateXCFilelists
if derived?
( $derivedSourcesPath + @path ) . to_s
else
'$(SRCROOT)/' + @path . to_s
end
elsif $mode == :GenerateBundles || ! derived?
2017-10-18 19:14:51 +00:00
@path . to_s
2017-09-26 15:34:19 +00:00
else
2017-10-18 19:14:51 +00:00
( $derivedSourcesPath + @path ) . to_s
2017-09-26 15:34:19 +00:00
end
end
2017-09-20 23:08:50 +00:00
end
2017-09-13 01:31:07 +00:00
class BundleManager
2019-01-10 02:20:13 +00:00
attr_reader :bundleCount , :extension , :fileCount , :currentBundleText , :maxCount , :extraFiles
2017-09-13 01:31:07 +00:00
2017-09-20 23:08:50 +00:00
def initialize ( extension , max )
2017-09-13 01:31:07 +00:00
@extension = extension
@fileCount = 0
@bundleCount = 0
@currentBundleText = " "
2017-09-20 23:08:50 +00:00
@maxCount = max
2019-01-10 02:20:13 +00:00
@extraFiles = [ ]
@currentDirectory = nil
2019-10-15 05:53:17 +00:00
@lastBundlingPrefix = nil
2017-09-13 01:31:07 +00:00
end
2017-10-18 19:14:51 +00:00
def writeFile ( file , text )
bundleFile = $unifiedSourceOutputPath + file
2018-11-28 22:07:45 +00:00
if $mode == :GenerateXCFilelists
$outputSources << bundleFile
return
end
2017-10-18 19:14:51 +00:00
if ( ! bundleFile . exist? || IO :: read ( bundleFile ) != @currentBundleText )
2018-08-24 22:54:23 +00:00
log ( " Writing bundle #{ bundleFile } with: \n #{ @currentBundleText } " )
2017-10-18 19:14:51 +00:00
IO :: write ( bundleFile , @currentBundleText )
end
end
2019-01-10 02:20:13 +00:00
def bundleFileName ( )
id =
if @maxCount
@bundleCount . to_s
else
# The dash makes the filenames more clear when using a hash.
hash = Digest :: SHA1 . hexdigest ( @currentDirectory . to_s ) [ 0 .. 7 ]
" - #{ hash } - #{ @bundleCount } "
end
@extension == " cpp " ? " UnifiedSource #{ id } . #{ extension } " : " UnifiedSource #{ id } - #{ extension } . #{ extension } "
2017-09-26 15:34:19 +00:00
end
2017-09-13 01:31:07 +00:00
def flush
@bundleCount += 1
2019-01-10 02:20:13 +00:00
bundleFile = bundleFileName
2017-10-18 19:14:51 +00:00
$generatedSources << $unifiedSourceOutputPath + bundleFile
2019-01-10 02:20:13 +00:00
@extraFiles << bundleFile if @maxCount and @bundleCount > @maxCount
2017-09-15 21:06:49 +00:00
2017-10-18 19:14:51 +00:00
writeFile ( bundleFile , @currentBundleText )
2017-09-13 01:31:07 +00:00
@currentBundleText = " "
@fileCount = 0
end
2017-10-18 19:14:51 +00:00
def flushToMax
raise if ! @maxCount
2019-01-10 02:20:13 +00:00
while @bundleCount < @maxCount
flush
end
2017-10-18 19:14:51 +00:00
end
def addFile ( sourceFile )
path = sourceFile . path
raise " wrong extension: #{ path . extname } expected #{ @extension } " unless path . extname == " . #{ @extension } "
2019-10-15 05:53:17 +00:00
bundlePrefix , bundleSize = BundlePrefixAndSizeForPath ( path )
if ( @lastBundlingPrefix != bundlePrefix )
2019-01-10 02:20:13 +00:00
log ( " Flushing because new top level directory; old: #{ @currentDirectory } , new: #{ path . dirname } " )
flush
2019-10-15 05:53:17 +00:00
@lastBundlingPrefix = bundlePrefix
2019-01-10 02:20:13 +00:00
@currentDirectory = path . dirname
@bundleCount = 0 unless @maxCount
end
2019-10-15 05:53:17 +00:00
if @fileCount > = bundleSize
2018-08-24 22:54:23 +00:00
log ( " Flushing because new bundle is full ( #{ @fileCount } sources) " )
2017-09-13 01:31:07 +00:00
flush
end
2017-10-18 19:14:51 +00:00
@currentBundleText += " # include \" #{ sourceFile } \" \n "
2017-09-13 01:31:07 +00:00
@fileCount += 1
end
end
2019-10-15 05:53:17 +00:00
def BundlePrefixAndSizeForPath ( path )
topLevelDirectory = TopLevelDirectoryForPath ( path . dirname )
$denseBundleFilters . each { | filter |
if path . fnmatch ( filter )
return filter , MAX_DENSE_BUNDLE_SIZE
end
}
return topLevelDirectory , MAX_BUNDLE_SIZE
end
2018-08-30 20:31:32 +00:00
def TopLevelDirectoryForPath ( path )
if ! path
return nil
end
while path . dirname != path . dirname . dirname
path = path . dirname
end
return path
end
2017-10-18 19:14:51 +00:00
def ProcessFileForUnifiedSourceGeneration ( sourceFile )
path = sourceFile . path
2018-11-28 22:07:45 +00:00
$inputSources << sourceFile . to_s
2017-09-13 01:31:07 +00:00
bundle = $bundleManagers [ path . extname ]
2018-08-24 22:54:23 +00:00
if ! bundle
log ( " No bundle for #{ path . extname } files, building #{ path } standalone " )
$generatedSources << sourceFile
elsif ! sourceFile . unifiable
log ( " Not allowed to unify #{ path } , building standalone " )
2017-10-18 19:14:51 +00:00
$generatedSources << sourceFile
2017-09-13 01:31:07 +00:00
else
2017-10-18 19:14:51 +00:00
bundle . addFile ( sourceFile )
2017-09-13 01:31:07 +00:00
end
2017-09-20 23:08:50 +00:00
end
$bundleManagers = {
" .cpp " = > BundleManager . new ( " cpp " , $maxCppBundleCount ) ,
" .mm " = > BundleManager . new ( " mm " , $maxObjCBundleCount )
}
2017-10-18 19:14:51 +00:00
seen = { }
sourceFiles = [ ]
2017-09-21 17:35:18 +00:00
2017-10-18 19:14:51 +00:00
sourceListFiles . each_with_index {
| path , sourceFileIndex |
2018-08-24 22:54:23 +00:00
log ( " Reading #{ path } " )
2017-10-18 19:14:51 +00:00
result = [ ]
File . read ( path ) . lines . each {
| line |
commentStart = line =~ COMMENT_REGEXP
2018-08-24 22:54:23 +00:00
log ( " Before: #{ line } " )
2017-10-18 19:14:51 +00:00
if commentStart != nil
line = line . slice ( 0 , commentStart )
2018-08-24 22:54:23 +00:00
log ( " After: #{ line } " )
2017-10-18 19:14:51 +00:00
end
line . strip!
2017-09-21 17:35:18 +00:00
2020-07-04 16:41:45 +00:00
next if line . empty?
2017-09-21 17:35:18 +00:00
2020-07-04 16:41:45 +00:00
if seen [ line ]
next if $mode == :GenerateXCFilelists
raise " duplicate line: #{ line } in #{ path } "
2017-09-21 17:35:18 +00:00
end
2020-07-04 16:41:45 +00:00
seen [ line ] = true
result << SourceFile . new ( line , sourceFileIndex )
2017-09-21 17:35:18 +00:00
}
2017-10-18 19:14:51 +00:00
2018-08-24 22:54:23 +00:00
log ( " Found #{ result . length } source files in #{ path } " )
2017-10-18 19:14:51 +00:00
sourceFiles += result
}
2017-09-20 23:08:50 +00:00
2017-10-18 19:14:51 +00:00
log ( " Found sources: #{ sourceFiles . sort } " )
sourceFiles . sort . each {
| sourceFile |
case $mode
2018-11-28 22:07:45 +00:00
when :GenerateBundles , :GenerateXCFilelists
2017-10-18 19:14:51 +00:00
ProcessFileForUnifiedSourceGeneration ( sourceFile )
2018-12-30 21:19:09 +00:00
when :PrintAllSources
$generatedSources << sourceFile
2017-10-18 19:14:51 +00:00
when :PrintBundledSources
$generatedSources << sourceFile if $bundleManagers [ sourceFile . path . extname ] && sourceFile . unifiable
end
2017-09-13 01:31:07 +00:00
}
2019-04-10 17:53:21 +00:00
if $mode != :PrintAllSources
$bundleManagers . each_value {
| manager |
manager . flush
2017-09-26 15:34:19 +00:00
2019-04-10 17:53:21 +00:00
maxCount = manager . maxCount
next if ! maxCount
2017-10-18 19:14:51 +00:00
2019-04-10 17:53:21 +00:00
manager . flushToMax
2019-01-10 02:20:13 +00:00
2019-04-10 17:53:21 +00:00
unless manager . extraFiles . empty?
extension = manager . extension
bundleCount = manager . bundleCount
filesToAdd = manager . extraFiles . join ( " , " )
raise " number of bundles for #{ extension } sources, #{ bundleCount } , exceeded limit, #{ maxCount } . Please add #{ filesToAdd } to Xcode then update UnifiedSource #{ extension . capitalize } FileCount "
end
}
end
2017-09-26 15:34:19 +00:00
2018-11-28 22:07:45 +00:00
if $mode == :GenerateXCFilelists
2019-02-05 22:58:19 +00:00
IO :: write ( $inputXCFilelistPath , $inputSources . sort . join ( " \n " ) + " \n " ) if $inputXCFilelistPath
IO :: write ( $outputXCFilelistPath , $outputSources . sort . join ( " \n " ) + " \n " ) if $outputXCFilelistPath
2018-11-28 22:07:45 +00:00
end
2017-09-13 01:31:07 +00:00
# We use stdout to report our unified source list to CMake.
2018-08-24 22:54:23 +00:00
# Add trailing semicolon and avoid a trailing newline for CMake's sake.
2017-09-26 15:34:19 +00:00
2017-10-18 19:14:51 +00:00
log ( $generatedSources . join ( " ; " ) + " ; " )
2017-09-13 01:31:07 +00:00
print ( $generatedSources . join ( " ; " ) + " ; " )