private final static int TIMEOUT_MILLIS = 100;

private final static int BLOCKING_TIMEOUT_SECONDS = 3000;

protected final JavaMode javaMode;

protected final Sketch sketch;

protected final ASTParser parser = ASTParser.newParser(AST.JLS11);

private final Thread preprocessingThread;

private final BlockingQueue<Boolean> requestQueue = new ArrayBlockingQueue<>(1);

private final Object requestLock = new Object();

private final AtomicBoolean codeFolderChanged = new AtomicBoolean(true);

private final AtomicBoolean librariesChanged = new AtomicBoolean(true);

private volatile boolean running;

private CompletableFuture<PreprocSketch> preprocessingTask = new CompletableFuture<>();

private CompletableFuture<?> lastCallback =

new CompletableFuture<>() {{

complete(null); // initialization block

}};

/**

public PreprocService(JavaMode javaMode, Sketch sketch) {

this.javaMode = javaMode;

this.sketch = sketch;

// Register listeners for first run

whenDone(this::fireListeners);

preprocessingThread = new Thread(this::mainLoop, "ECS");

preprocessingThread.start();

}

/**

private void mainLoop() {

running = true;

PreprocSketch prevResult = null;

CompletableFuture<?> runningCallbacks = null;

Messages.log("Hi!");

while (running) {

try {

try {

requestQueue.take(); // blocking until requested

} catch (InterruptedException e) {

running = false;

break;

}

Messages.log("Starting");

prevResult = preprocessSketch(prevResult);

// Wait until callbacks finish before firing new wave

// If new request arrives while waiting, break out and start preprocessing

while (requestQueue.isEmpty() && runningCallbacks != null) {

try {

runningCallbacks.get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);

runningCallbacks = null;

} catch (TimeoutException ignored) { }

}

synchronized (requestLock) {

if (requestQueue.isEmpty()) {

runningCallbacks = lastCallback;

Messages.log("Done");

preprocessingTask.complete(prevResult);

}

}

} catch (Exception e) {

Messages.err("problem in preprocessor service loop", e);

}

}

Messages.log("Bye!");

}

/**

public void dispose() {

cancel();

running = false;

preprocessingThread.interrupt();

}

/**

public void cancel() {

requestQueue.clear();

}

/**

public void notifySketchChanged() {

synchronized (requestLock) {

if (preprocessingTask.isDone()) {

preprocessingTask = new CompletableFuture<>();

// Register callback which executes all listeners

whenDone(this::fireListeners);

}

requestQueue.offer(Boolean.TRUE);

}

}

/**

public void notifyLibrariesChanged() {

Messages.log("notified libraries changed");

librariesChanged.set(true);

notifySketchChanged();

}

/**

public void notifyCodeFolderChanged() {

Messages.log("notified code folder changed");

codeFolderChanged.set(true);

notifySketchChanged();

}

/**

private CompletableFuture<?> registerCallback(Consumer<PreprocSketch> callback) {

synchronized (requestLock) {

lastCallback = preprocessingTask

// Run callback after both preprocessing task and previous callback

.thenAcceptBothAsync(lastCallback, (ps, a) -> callback.accept(ps))

// Make sure exception in callback won't cancel whole callback chain

.handleAsync((res, e) -> {

if (e != null) Messages.err("exception in callback", e);

return res;

});

return lastCallback;

}

}

/**

public void whenDone(Consumer<PreprocSketch> callback) {

registerCallback(callback);

}

/**

public void whenDoneBlocking(Consumer<PreprocSketch> callback) {

try {

registerCallback(callback).get(BLOCKING_TIMEOUT_SECONDS, TimeUnit.SECONDS);

} catch (InterruptedException | ExecutionException | TimeoutException e) {

// Don't care

}

}

/// LISTENERS ----------------------------------------------------------------

private final Set<Consumer<PreprocSketch>> listeners = new CopyOnWriteArraySet<>();

/**

public void registerListener(Consumer<PreprocSketch> listener) {

if (listener != null) listeners.add(listener);

}

/**

public void unregisterListener(Consumer<PreprocSketch> listener) {

listeners.remove(listener);

}

/**

private void fireListeners(PreprocSketch ps) {

for (Consumer<PreprocSketch> listener : listeners) {

try {

listener.accept(ps);

} catch (Exception e) {

Messages.err("error when firing preprocessing listener", e);

}

}

}

/// --------------------------------------------------------------------------

/**

private PreprocSketch preprocessSketch(PreprocSketch prevResult) {

boolean firstCheck = (prevResult == null);

PreprocSketch.Builder result = new PreprocSketch.Builder();

List<ImportStatement> codeFolderImports = result.codeFolderImports;

List<ImportStatement> programImports = result.programImports;

result.sketch = this.sketch;

String className = sketch.getMainName();

StringBuilder workBuffer = new StringBuilder();

// Combine code into one buffer

int numLines = 1;

IntList tabStartsList = new IntList();

List<JavaSketchCode> javaFiles = new ArrayList<>();

List<Integer> tabLineStarts = new ArrayList<>();

for (SketchCode sc : sketch.getCode()) {

if (sc.isExtension("pde")) {

tabStartsList.append(workBuffer.length());

tabLineStarts.add(numLines);

String newPieceBuilt = getSketchTabContents(sc) + '\n';

numLines += SourceUtil.getCount(newPieceBuilt, "\n");

workBuffer.append(newPieceBuilt);

} else if (sc.isExtension("java")) {

tabStartsList.append(workBuffer.length());

tabLineStarts.add(numLines);

javaFiles.add(new JavaSketchCode(sc, tabStartsList.size()-1));

String newPieceBuilt = String.format(

"\n// Skip java file %s.\n",

sc.getFileName()

);

numLines += SourceUtil.getCount(newPieceBuilt, "\n");

workBuffer.append(newPieceBuilt);

}

}

result.tabStartOffsets = tabStartsList.toArray();

String pdeStage = result.pdeCode = workBuffer.toString();

boolean reloadCodeFolder = firstCheck || codeFolderChanged.getAndSet(false);

boolean reloadLibraries = firstCheck || librariesChanged.getAndSet(false);

// Core and default imports

PdePreprocessor preProcessor =

PdePreprocessor.builderFor(this.sketch.getMainName()).build();

if (coreAndDefaultImports == null) {

coreAndDefaultImports = buildCoreAndDefaultImports(preProcessor);

}

result.coreAndDefaultImports.addAll(coreAndDefaultImports);

// Prepare code folder imports

if (reloadCodeFolder) {

codeFolderImports.addAll(buildCodeFolderImports(sketch));

} else {

codeFolderImports.addAll(prevResult.codeFolderImports);

}

// TODO: convert unicode escapes to chars

// TODO: This appears no longer to be needed.

SourceUtil.scrubCommentsAndStrings(workBuffer);

result.scrubbedPdeCode = workBuffer.toString();

PreprocessorResult preprocessorResult;

try {

preprocessorResult = preProcessor.write(

new StringWriter(),

result.scrubbedPdeCode,

codeFolderImports.stream()

.map(ImportStatement::getFullMemberName)

.collect(Collectors.toList())

);

} catch (SketchException e) {

throw new RuntimeException("Unexpected sketch exception in preprocessing: " + e);

}

final int endNumLines = numLines;

if (preprocessorResult.getPreprocessIssues().size() > 0) {

preprocessorResult.getPreprocessIssues().stream()

.map((x) -> ProblemFactory.build(x, tabLineStarts))

.forEach(result.otherProblems::add);

result.hasSyntaxErrors = true;

}

// Save off the imports

programImports.addAll(preprocessorResult.getImportStatements());

result.programImports.addAll(preprocessorResult.getImportStatements());

// Prepare transforms to convert pde code into parsable code

TextTransform toParsable = new TextTransform(pdeStage);

toParsable.addAll(preprocessorResult.getEdits());

{ // Refresh sketch classloader and classpath if imports changed

if (reloadLibraries) {

runtimePathBuilder.markLibrariesChanged();

}

boolean rebuildLibraryClassPath = reloadLibraries ||

checkIfImportsChanged(programImports, prevResult.programImports);

if (rebuildLibraryClassPath) {

runtimePathBuilder.markLibraryImportsChanged();

}

boolean rebuildClassPath = reloadCodeFolder || rebuildLibraryClassPath ||

prevResult.classLoader == null || prevResult.classPath == null ||

prevResult.classPathArray == null || prevResult.searchClassPathArray == null;

if (reloadCodeFolder) {

runtimePathBuilder.markCodeFolderChanged();

}

if (rebuildClassPath) {

runtimePathBuilder.prepareClassPath(result, javaMode);

} else {

result.classLoader = prevResult.classLoader;

result.classPath = prevResult.classPath;

result.searchClassPathArray = prevResult.searchClassPathArray;

result.classPathArray = prevResult.classPathArray;

}

}

// Transform code to parsable state

String parsableStage = toParsable.apply();

OffsetMapper parsableMapper = toParsable.getMapper();

// Create intermediate AST for advanced preprocessing

// Wait on .java tabs due to speed since they don't go through preproc.

CompileResults parsableCompile = compileInMemory(

parsableStage,

className,

result.classPathArray,

false

);

CompilationUnit parsableCU = parsableCompile.getCompilationUnit();

// Prepare advanced transforms which operate on AST

TextTransform toCompilable = new TextTransform(parsableStage);

toCompilable.addAll(SourceUtil.preprocessAST(parsableCU));

// Transform code to compilable state

// Again, wait on .java tabs due to speed since they don't go through

// the preprocessor.

String compilableStage = toCompilable.apply();

OffsetMapper compilableMapper = toCompilable.getMapper();

// Create compilable AST to get syntax problems

CompileResults compilableCompile = compileInMemory(

compilableStage,

className,

result.classPathArray,

false

);

CompilationUnit compilableCU = compilableCompile.getCompilationUnit();

// Get syntax problems from compilable AST

result.hasSyntaxErrors |=

Arrays.stream(compilableCU.getProblems()).anyMatch(IProblem::isError);

// Generate bindings after getting problems - avoids

// 'missing type' errors when there are syntax problems.

// Introduce .java tabs here for type resolution.

CompileResults bindingsCompile;

if (javaFiles.size() == 0) {

bindingsCompile = compileInMemory(

compilableStage,

className,

result.classPathArray,

true

);

} else {

bindingsCompile = compileFromDisk(

compilableStage,

className,

result.classPathArray,

javaFiles

);

}

// Get compilation problems

List<IProblem> bindingsProblems = bindingsCompile.getProblems();

result.hasCompilationErrors =

bindingsProblems.stream().anyMatch(IProblem::isError);

// Update builder

result.offsetMapper = parsableMapper.thenMapping(compilableMapper);

result.javaCode = compilableStage;

result.compilationUnit = bindingsCompile.getCompilationUnit();

result.javaFileMapping = bindingsCompile.getJavaFileMapping();

result.iproblems = bindingsCompile.getProblems();

// Build it

return result.build();

}

/**

private String getSketchTabContents(SketchCode sketchCode) {

String code = null;

if (sketchCode.getDocument() != null) {

try {

code = sketchCode.getDocumentText();

} catch (BadLocationException e) {

e.printStackTrace();

}

}

if (code == null) {

code = sketchCode.getProgram();

}

return code;

}

/// COMPILATION -----------------------------------------------------------

/**

private CompileResults compileInMemory(String sketchSource, String className,

String[] classPathArray, boolean resolveBindings) {

// Prepare parser

parser.setSource(sketchSource.toCharArray());

setupParser(resolveBindings, className, classPathArray);

CompilationUnit compilationUnit = (CompilationUnit) parser.createAST(null);

List<IProblem> problems = Arrays.asList(compilationUnit.getProblems());

return new CompileResults(compilationUnit, problems, new HashMap<>());

}

/**

private CompileResults compileFromDisk(String sketchSource,

String className, String[] classPathArray,

List<JavaSketchCode> javaFiles) {

ProcessingASTRequester astRequester;

List<Path> temporaryFilesList = new ArrayList<>();

Map<String, Integer> javaFileMapping = new HashMap<>();

// Prepare parser

setupParser(true, className, classPathArray);

// Write compilable processing file

Path mainTemporaryFile = createTemporaryFile(sketchSource);

final String mainSource = mainTemporaryFile.toString();

temporaryFilesList.add(mainTemporaryFile);

// Write temporary java files as tab may be unsaved

for (JavaSketchCode javaFile : javaFiles) {

String tabContents = getSketchTabContents(javaFile.getSketchCode());

Path newPath = createTemporaryFile(tabContents);

javaFileMapping.put(newPath.toString(), javaFile.getTabIndex());

temporaryFilesList.add(newPath);

}

// Convert paths

int numFiles = temporaryFilesList.size();

String[] temporaryFilesArray = new String[numFiles];

for (int i = 0; i < numFiles; i++) {

temporaryFilesArray[i] = temporaryFilesList.get(i).toString();

}

// Compile

astRequester = new ProcessingASTRequester(mainSource);

parser.createASTs(

temporaryFilesArray,

null,

new String[] {},

astRequester,

null

);

// Delete

deleteTemporaryFiles(temporaryFilesList);

// Return

return new CompileResults(

astRequester.getMainCompilationUnit(),

astRequester.getProblems(),

javaFileMapping

);

}

/**

private void setupParser(boolean resolveBindings, String className,

String[] classPathArray) {

parser.setKind(ASTParser.K_COMPILATION_UNIT);

parser.setCompilerOptions(COMPILER_OPTIONS);

parser.setStatementsRecovery(true);

if (resolveBindings) {

parser.setUnitName(className);

parser.setEnvironment(classPathArray, null, null, false);

parser.setResolveBindings(true);

}

}

/**

private Path createTemporaryFile(String content) {

try {

Path tempFile = Files.createTempFile(null, null);

Files.writeString(tempFile, content);

return tempFile;

} catch (IOException e) {

throw new RuntimeException("Cannot write to temporary folder.");

}

}

/**

private void deleteTemporaryFiles(List<Path> paths) {

try {

for (Path path : paths) {

Files.delete(path);

}

} catch (IOException e) {

throw new RuntimeException("Cannot delete from temporary folder.");

}

}

/**

static private class ProcessingASTRequester extends FileASTRequestor {

private final String mainSource;

private final List<IProblem> problems;

private CompilationUnit mainCompilationUnit;

/**

public ProcessingASTRequester(String newMainSource) {

mainSource = newMainSource;

problems = new ArrayList<>();

}

/**

public void acceptAST(String source, CompilationUnit ast) {

if (source.equals(mainSource)) {

mainCompilationUnit = ast;

}

Collections.addAll(problems, ast.getProblems());

}

/**

public List<IProblem> getProblems() {

return problems;

}

/**

public CompilationUnit getMainCompilationUnit() {

return mainCompilationUnit;

}

}

/**

static private class CompileResults {

private final CompilationUnit compilationUnit;

private final List<IProblem> problems;

private final Map<String, Integer> javaFileMapping;

/**

public CompileResults(CompilationUnit newCompilationUnit,

List<IProblem> newProblems, Map<String, Integer> newJavaFileMapping) {

compilationUnit = newCompilationUnit;

problems = newProblems;

javaFileMapping = newJavaFileMapping;

}

/**

public CompilationUnit getCompilationUnit() {

return compilationUnit;

}

/**

public List<IProblem> getProblems() {

return problems;

}

/**

public Map<String, Integer> getJavaFileMapping() {

return javaFileMapping;

}

}

/**

static private class JavaSketchCode {

private final SketchCode sketchCode;

private final int tabIndex;

/**

public JavaSketchCode(SketchCode newSketchCode, int newTabIndex) {

sketchCode = newSketchCode;

tabIndex = newTabIndex;

}

/**

public SketchCode getSketchCode() {

return sketchCode;

}

/**

public int getTabIndex() {

return tabIndex;

}

}

/// IMPORTS -----------------------------------------------------------------

private List<ImportStatement> coreAndDefaultImports;

/**

private static List<ImportStatement> buildCoreAndDefaultImports(PdePreprocessor preprocessor) {

List<ImportStatement> result = new ArrayList<>();

for (String imp : preprocessor.getCoreImports()) {

result.add(ImportStatement.parse(imp));

}

for (String imp : preprocessor.getDefaultImports()) {

result.add(ImportStatement.parse(imp));

}

return result;

}

/**

private static List<ImportStatement> buildCodeFolderImports(Sketch sketch) {

if (sketch.hasCodeFolder()) {

File codeFolder = sketch.getCodeFolder();

String codeFolderClassPath = Util.contentsToClassPath(codeFolder);

StringList codeFolderPackages = Util.packageListFromClassPath(codeFolderClassPath);

return StreamSupport.stream(codeFolderPackages.spliterator(), false)

.map(ImportStatement::wholePackage)

.collect(Collectors.toList());

}

return Collections.emptyList();

}

/**

private static boolean checkIfImportsChanged(List<ImportStatement> prevImports,

List<ImportStatement> imports) {

if (imports.size() != prevImports.size()) {

return true;

} else {

int count = imports.size();

for (int i = 0; i < count; i++) {

if (!imports.get(i).isSameAs(prevImports.get(i))) {

return true;

}

}

}

return false;

}

/// CLASS PATHS --------------------------------------------------------------

private final RuntimePathBuilder runtimePathBuilder = new RuntimePathBuilder();

/// --------------------------------------------------------------------------

static private final Map<String, String> COMPILER_OPTIONS;

static {

Map<String, String> compilerOptions = new HashMap<>();

// TODO: VERSION_17 if using new language features. Requires new JDT.

compilerOptions.put(JavaCore.COMPILER_COMPLIANCE, JavaCore.VERSION_11);

compilerOptions.put(JavaCore.COMPILER_SOURCE, JavaCore.VERSION_11);

compilerOptions.put(JavaCore.COMPILER_CODEGEN_TARGET_PLATFORM, JavaCore.VERSION_11);

// See http://help.eclipse.org/mars/index.jsp?topic=%2Forg.eclipse.jdt.doc.isv%2Fguide%2Fjdt_api_options.htm&anchor=compiler

final String[] generate = {

JavaCore.COMPILER_LINE_NUMBER_ATTR,

JavaCore.COMPILER_SOURCE_FILE_ATTR

};

final String[] ignore = {

JavaCore.COMPILER_PB_UNUSED_IMPORT,

JavaCore.COMPILER_PB_MISSING_SERIAL_VERSION,

JavaCore.COMPILER_PB_RAW_TYPE_REFERENCE,

JavaCore.COMPILER_PB_REDUNDANT_TYPE_ARGUMENTS,

JavaCore.COMPILER_PB_UNCHECKED_TYPE_OPERATION

};

final String[] warn = {

JavaCore.COMPILER_PB_NO_EFFECT_ASSIGNMENT,

JavaCore.COMPILER_PB_NULL_REFERENCE,

JavaCore.COMPILER_PB_POTENTIAL_NULL_REFERENCE,

JavaCore.COMPILER_PB_REDUNDANT_NULL_CHECK,

JavaCore.COMPILER_PB_POSSIBLE_ACCIDENTAL_BOOLEAN_ASSIGNMENT,

JavaCore.COMPILER_PB_UNUSED_LABEL,

JavaCore.COMPILER_PB_UNUSED_LOCAL,

JavaCore.COMPILER_PB_UNUSED_OBJECT_ALLOCATION,

JavaCore.COMPILER_PB_UNUSED_PARAMETER,

JavaCore.COMPILER_PB_UNUSED_PRIVATE_MEMBER

};

for (String s : generate) compilerOptions.put(s, JavaCore.GENERATE);

for (String s : ignore) compilerOptions.put(s, JavaCore.IGNORE);

for (String s : warn) compilerOptions.put(s, JavaCore.WARNING);

COMPILER_OPTIONS = Collections.unmodifiableMap(compilerOptions);

}