1 # LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
\r
2 # Released to the public domain.
\r
4 # Regenerate the Scintilla and SciTE source files that list
\r
5 # all the lexers and all the properties files.
\r
6 # Should be run whenever a new lexer is added or removed.
\r
7 # Requires Python 2.4 or later
\r
8 # Most files are regenerated in place with templates stored in comments.
\r
9 # The VS .NET project file is generated into a different file as the
\r
10 # VS .NET environment will not retain comments when modifying the file.
\r
11 # The files are copied to a string apart from sections between a
\r
12 # ++Autogenerated comment and a --Autogenerated comment which is
\r
13 # generated by the CopyWithInsertion function. After the whole
\r
14 # string is instantiated, it is compared with the target file and
\r
15 # if different the file is rewritten.
\r
16 # Does not regenerate the Visual C++ 6 project files but does the VS .NET
\r
28 if sys.platform == "win32":
\r
31 # Yes, LF is the native EOL even on Mac OS X. CR is just for
\r
32 # Mac OS <=9 (a.k.a. "Mac Classic")
\r
35 # Automatically generated sections contain start and end comments,
\r
36 # a definition line and the results.
\r
37 # The results are replaced by regenerating based on the definition line.
\r
38 # The definition line is a comment prefix followed by "**".
\r
39 # If there is a digit after the ** then this indicates which list to use
\r
40 # and the digit and next character are not part of the definition
\r
41 # Backslash is used as an escape within the definition line.
\r
42 # The part between \( and \) is repeated for each item in the list.
\r
43 # \* is replaced by each list item. \t, and \n are tab and newline.
\r
44 def CopyWithInsertion(input, commentPrefix, retainDefs, eolType, *lists):
\r
48 for line in input.splitlines(0):
\r
49 isStartGenerated = line.startswith(commentPrefix + "++Autogenerated")
\r
50 if copying and not isStartGenerated:
\r
52 if isStartGenerated:
\r
57 elif not copying and line.startswith(commentPrefix + "**"):
\r
60 definition = line[len(commentPrefix + "**"):]
\r
62 if definition[0] in string.digits:
\r
63 listid = int(definition[:1])
\r
64 definition = definition[2:]
\r
65 # Hide double slashes as a control character
\r
66 definition = definition.replace("\\\\", "\001")
\r
67 # Do some normal C style transforms
\r
68 definition = definition.replace("\\n", "\n")
\r
69 definition = definition.replace("\\t", "\t")
\r
70 # Get the doubled backslashes back as single backslashes
\r
71 definition = definition.replace("\001", "\\")
\r
72 startRepeat = definition.find("\\(")
\r
73 endRepeat = definition.find("\\)")
\r
74 intro = definition[:startRepeat]
\r
76 if intro.endswith("\n"):
\r
81 middle = definition[startRepeat+2:endRepeat]
\r
82 for i in lists[listid]:
\r
83 item = middle.replace("\\*", i)
\r
84 if pos and (pos + len(item) >= 80):
\r
89 if item.endswith("\n"):
\r
91 outro = definition[endRepeat+2:]
\r
93 out = out.replace("\n", eolType) # correct EOLs in generated content
\r
95 elif line.startswith(commentPrefix + "--Autogenerated"):
\r
99 output = [line.rstrip(" \t") for line in output] # trim trailing whitespace
\r
100 return eolType.join(output) + eolType
\r
102 def UpdateFile(filename, updated):
\r
103 """ If the file is different to updated then copy updated
\r
104 into the file else leave alone so CVS and make don't treat
\r
105 it as modified. """
\r
107 infile = open(filename, "rb")
\r
108 except IOError: # File is not there yet
\r
109 out = open(filename, "wb")
\r
112 print "New", filename
\r
114 original = infile.read()
\r
116 if updated != original:
\r
117 os.unlink(filename)
\r
118 out = open(filename, "wb")
\r
121 print "Changed", filename
\r
123 #~ print "Unchanged", filename
\r
125 def Generate(inpath, outpath, commentPrefix, eolType, *lists):
\r
126 """Generate 'outpath' from 'inpath'.
\r
128 "eolType" indicates the type of EOLs to use in the generated
\r
129 file. It should be one of following constants: LF, CRLF,
\r
132 #print "generate '%s' -> '%s' (comment prefix: %r, eols: %r)"\
\r
133 # % (inpath, outpath, commentPrefix, eolType)
\r
135 infile = open(inpath, "r")
\r
137 print "Can not open", inpath
\r
139 original = infile.read()
\r
141 updated = CopyWithInsertion(original, commentPrefix,
\r
142 inpath == outpath, eolType, *lists)
\r
143 UpdateFile(outpath, updated)
\r
145 def Regenerate(filename, commentPrefix, eolType, *lists):
\r
146 """Regenerate the given file.
\r
148 "eolType" indicates the type of EOLs to use in the generated
\r
149 file. It should be one of following constants: LF, CRLF,
\r
152 Generate(filename, filename, commentPrefix, eolType, *lists)
\r
154 def FindModules(lexFile):
\r
157 for l in f.readlines():
\r
158 if l.startswith("LexerModule"):
\r
159 l = l.replace("(", " ")
\r
160 modules.append(l.split()[1])
\r
163 knownIrregularProperties = [
\r
165 "styling.within.preprocessor",
\r
166 "tab.timmy.whinge.level",
\r
167 "asp.default.language",
\r
168 "html.tags.case.sensitive",
\r
171 "sql.backslash.escapes",
\r
176 def FindProperties(lexFile):
\r
179 for l in f.readlines():
\r
180 if "GetProperty" in l:
\r
182 if not l.startswith("//"): # Drop comments
\r
183 propertyName = l.split("\"")[1]
\r
184 if propertyName.lower() == propertyName:
\r
185 # Only allow lower case property names
\r
186 if propertyName in knownIrregularProperties or \
\r
187 propertyName.startswith("fold.") or \
\r
188 propertyName.startswith("lexer."):
\r
189 properties.add(propertyName)
\r
192 def ciCompare(a,b):
\r
193 return cmp(a.lower(), b.lower())
\r
195 def RegenerateAll():
\r
198 # Find all the lexer source code files
\r
199 lexFilePaths = glob.glob(root + "scintilla/src/Lex*.cxx")
\r
200 lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths]
\r
203 lexerProperties = set()
\r
204 for lexFile in lexFilePaths:
\r
205 lexerModules.extend(FindModules(lexFile))
\r
206 lexerProperties.update(FindProperties(lexFile))
\r
207 lexerModules.sort(ciCompare)
\r
208 lexerProperties.remove("fold.comment.python")
\r
209 lexerProperties = list(lexerProperties)
\r
210 lexerProperties.sort(ciCompare)
\r
212 # Find all the SciTE properties files
\r
213 otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"]
\r
214 if os.path.exists(root + "scite"):
\r
215 propFilePaths = glob.glob(root + "scite/src/*.properties")
\r
216 propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps]
\r
217 propFiles.sort(ciCompare)
\r
220 Regenerate(root + "scintilla/src/KeyWords.cxx", "//", NATIVE, lexerModules)
\r
221 Regenerate(root + "scintilla/win32/makefile", "#", NATIVE, lexFiles)
\r
222 Regenerate(root + "scintilla/win32/scintilla.mak", "#", NATIVE, lexFiles)
\r
223 Regenerate(root + "scintilla/win32/scintilla_vc6.mak", "#", NATIVE, lexFiles)
\r
224 # Use Unix EOLs for gtk Makefiles so they work for Linux users when
\r
225 # extracted from the Scintilla source ZIP (typically created on
\r
227 Regenerate(root + "scintilla/gtk/makefile", "#", LF, lexFiles)
\r
228 Regenerate(root + "scintilla/gtk/scintilla.mak", "#", NATIVE, lexFiles)
\r
229 Regenerate(root + "scintilla/macosx/makefile", "#", LF, lexFiles)
\r
230 if os.path.exists(root + "scite"):
\r
231 Regenerate(root + "scite/win32/makefile", "#", NATIVE, lexFiles, propFiles)
\r
232 Regenerate(root + "scite/win32/scite.mak", "#", NATIVE, lexFiles, propFiles)
\r
233 Regenerate(root + "scite/src/SciTEProps.cxx", "//", NATIVE, lexerProperties)
\r
234 Generate(root + "scite/boundscheck/vcproj.gen",
\r
235 root + "scite/boundscheck/SciTE.vcproj", "#", NATIVE, lexFiles)
\r