如何构建回测系统（十）

问题

在（九）中实现了一个可以从数据库中编译并加载代码的classloader，但这是有缺陷的

缺陷在于，如果要使用回测系统实现的类和对象则会出现编译无法通过

提示Symbol找不到或者缺失

定位问题

日常我们使用编译功能一般都是maven或者是ant等框架来编译，很少说直接用javac来直接对整个项目编译

在（九）使用的compiler其实就是javac的java代码版本如果采用javac编译一个引用了其他java或者class的文件，则会出现很类似的症状

找到Maven/IDE等工具具体执行javac的代码其实就能看到，是编译时候带上了参数-cp/-classpath/

解决问题

compiler的getTask函数中有个参数叫options，根据文档，这是类似于javac中的参数，但其实如果我们简单的把依赖的class按照-cp的形式添加到参数中就会发现直接报错

这是因为-cp是传给jvm的参数，getTask函数只支持具体的54个参数，属于编译期间具体的信息数据用的

而-cp产生作用的方式是在FileManager中的list()函数起作用的（这里完全没有任何文档，只能单步调试代码发现，所以这里无法说以后就一定正确）

我们只需要将依赖的class/jar/java文件按照需求变更成SimpleJavaFileObject既可

参考代码

因为环境因素，这里的参考代码是在springboot中实现的

package com.michaelssss.domain.strategy;

import static javax.tools.StandardLocation.CLASS_PATH;

import com.michaelssss.exception.RunException;
import com.michaelssss.utils.ArrayUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.tools.DiagnosticCollector;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
import javax.tools.JavaCompiler;
import javax.tools.JavaFileManager;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import javax.tools.SimpleJavaFileObject;
import javax.tools.ToolProvider;
import org.springframework.stereotype.Component;

@Component
public class MyCompiler {

    public byte[] compile(String name, String codes) {
        JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
        var diagnosticCollector = new DiagnosticCollector<JavaFileObject>();
        var fileManager = new MyJavaFileObjectManager(
            compiler.getStandardFileManager(diagnosticCollector, Locale.CHINESE,
                StandardCharsets.UTF_8));
        fileManager.complieName = name;
        var javaFileObjects = Collections
            .singletonList(
                new MyJavaFileObject(name.replace('.', '/'), codes, fileManager.outputStream));
        var options = new ArrayList<String>();
        var compilerTask = compiler
            .getTask(null, fileManager, diagnosticCollector, options, null, javaFileObjects);
        var result = compilerTask.call();
        if (Objects.nonNull(result) && result) {
            return fileManager.outputStream.toByteArray();
        } else {
            StringBuilder stringBuilder = new StringBuilder();
            diagnosticCollector.getDiagnostics()
                .forEach(diagnostic -> stringBuilder.append(
                    diagnostic.toString()
                ).append("\r\n"));
            throw new RunException(stringBuilder.toString());
        }
    }

    static class MyJavaFileObjectManager extends ForwardingJavaFileManager<JavaFileManager> {

        Map<String, JavaFileObject> fileObjects = new HashMap<>();
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        List<JavaFileObject> result;
        String complieName;

        public MyJavaFileObjectManager(JavaFileManager fileManager) {
            super(fileManager);
        }

        @Override
        public JavaFileObject getJavaFileForInput(Location location, String className,
            Kind kind) throws IOException {
            JavaFileObject javaFileObject = fileObjects.get(className);
            if (!location.getName().contains(CLASS_PATH.getName())) {
                return super.getJavaFileForInput(location, className, kind);
            }
            if (Objects.isNull(javaFileObject)) {
                if (Objects.isNull(result)) {
                    result = CompilerDependenceResolver.resolve();
                }
                var aa = result.stream().filter(
                    f -> f.getName().equalsIgnoreCase(className) && kind.equals(f.getKind()))
                    .collect(
                        Collectors.toList());
                if (aa.isEmpty()) {
                    return super.getJavaFileForInput(location, className, kind);
                } else {
                    return aa.get(0);
                }
            }
            return javaFileObject;
        }

        @Override
        public JavaFileObject getJavaFileForOutput(Location location, String qualifiedClassName,
            JavaFileObject.Kind kind, FileObject sibling) throws IOException {
            JavaFileObject javaFileObject = new MyJavaFileObject(qualifiedClassName, kind,
                outputStream);
            fileObjects.put(qualifiedClassName, javaFileObject);
            return javaFileObject;
        }

        @Override
        public String inferBinaryName(Location location, JavaFileObject file) {
            if (file instanceof MyJavaFileObject) {
                return file.getName();
            }
            if (file instanceof WithPackageAccessSimplaJava) {
                return ((WithPackageAccessSimplaJava) file).getBinaryName();
            }
            if (
                file instanceof NewJarJavaFileObject
            ) {
                return ((NewJarJavaFileObject) file).getBinaryName();
            } else {
                return super.inferBinaryName(location, file);
            }
        }

        @Override
        public Iterable<JavaFileObject> list(Location location, String packageName, Set<Kind> kinds,
            boolean recurse) throws IOException {

            if (location.getName().contains(CLASS_PATH.getName())) {
                if (Objects.isNull(result)) {
                    result = CompilerDependenceResolver.resolve();
                }
                if (!result.isEmpty()) {
                    var a = result.stream()
                        .filter(
                            r -> ArrayUtils
                                .isContain(((MySimpleJavaObjectI) r).getBinaryName().split("\\."),
                                    packageName.split("\\.")) && kinds
                                .contains(r.getKind()))
                        .collect(Collectors.toList());
                    var e = super.list(location, packageName, kinds, recurse);
                    e.forEach(a::add);
                    return a;
                }
            }
            return super.list(location, packageName, kinds, recurse);
        }
    }


    static class MyJavaFileObject extends SimpleJavaFileObject {

        String source;
        OutputStream outputStream;
        String name;

        public MyJavaFileObject(String name, Kind kind, OutputStream stream) {
            super(URI.create("String:///" + name + kind.extension), kind);
            this.outputStream = stream;
        }

        public MyJavaFileObject(String name, String source) {
            super(URI.create("String:///" + name + Kind.SOURCE.extension), Kind.SOURCE);
            this.source = source;
            this.name = name;
        }

        public MyJavaFileObject(String name, String source, OutputStream outputStream) {
            this(name, source);
            this.outputStream = outputStream;
        }

        public MyJavaFileObject(URI uri, Kind kind) {
            super(uri, kind);
        }

        @Override
        public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {
            return source;
        }

        @Override
        public OutputStream openOutputStream() throws IOException {
            return outputStream;
        }
    }
}

package com.michaelssss.domain.strategy;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.boot.system.ApplicationHome;

public class CompilerDependenceResolver {

    /**
     * @return 返回所有非SystemLoader加载的class的JavaFileObject
     */
    public static List<String> getClassPath() {
        var systemType = System.getProperty("os.name");

        var classpath = new String[0];
        if (systemType.contains("Windows")) {
            classpath = System.getProperty("java.class.path").split(";");
        }
        if (systemType.contains("Linux")) {
            classpath = System.getProperty("java.class.path").split(":");
        }
        return Arrays.asList(classpath);
    }

    @SneakyThrows
    public static List<JavaFileObject> resolve() {
        List<JavaFileObject> result = new ArrayList<>();
        //查询所有非springboot主文件本身的jar
        final List<String> jarlibs = new ArrayList<>();
        var loadPath = System.getProperty("loader.path");
        var classpath = System.getProperty("java.class.path").split(";");
        if (StringUtils.isNotEmpty(loadPath)) {
            classpath = Files.list(Path.of(loadPath))
                .filter(path -> !Files.isDirectory(path) && path.toString().endsWith(".jar"))
                .map(Path::toString).toArray(String[]::new);
        }
        jarlibs.addAll(Arrays.stream(classpath).filter(s -> s.endsWith(".jar"))
            .collect(Collectors.toList()));
        var compilerModuleClassPaths = Arrays.stream(classpath).filter(s -> !s.endsWith(".jar"))
            .collect(Collectors.toList());

        if (jarlibs.isEmpty()) {
            throw new Error("load jar info failed");
        }
        jarlibs.forEach(s -> {
            try {
                JarFile jarFile = new JarFile(s);
                jarFile
                    .stream()
                    .forEach(jarEntry -> {
                        if (!jarEntry.isDirectory() && jarEntry.getName().endsWith(".class")) {
                            try (var in = jarFile.getInputStream(jarEntry)) {
                                var jarPath = Path.of(s);
                                String separator = jarEntry.getName().startsWith("/") ? "!" : "!/";
                                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
                                IOUtils.copy(in, outputStream);
                                result.add(
                                    new NewJarJavaFileObject(
                                        URI.create(
                                            "jar:///" + jarPath.toUri().normalize() + separator
                                                + jarEntry
                                                .getName()),
                                        Kind.CLASS,
                                        outputStream,
                                        jarEntry
                                    ));
                            } catch (IOException e) {
                                e.printStackTrace();
                            }
                        }
                    });

            } catch (IOException e) {
                e.printStackTrace();
            }

        });
        if (!isJar()) {
            var path = Path.of(getPath());
            Files.walkFileTree(path, new FileVisitor<Path>() {
                @Override
                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
                    throws IOException {
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                    throws IOException {
                    if (file.toString().endsWith(".class")) {

                        WithPackageAccessSimplaJava withPackageAccessSimplaJava = new WithPackageAccessSimplaJava(
                            path.toUri(),
                            Kind.CLASS, path.normalize().toString(), file);
                        result.add(withPackageAccessSimplaJava);
                    }
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFileFailed(Path file, IOException exc)
                    throws IOException {
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult postVisitDirectory(Path dir, IOException exc)
                    throws IOException {
                    return FileVisitResult.CONTINUE;
                }
            });
            compilerModuleClassPaths.forEach(path1 -> {
                try {
                    Files.walkFileTree(Path.of(path1), new FileVisitor<Path>() {
                        @Override
                        public FileVisitResult preVisitDirectory(Path dir,
                            BasicFileAttributes attrs)
                            throws IOException {
                            return FileVisitResult.CONTINUE;
                        }

                        @Override
                        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                            throws IOException {
                            if (file.toString().endsWith(".class")) {
                                WithPackageAccessSimplaJava withPackageAccessSimplaJava = new WithPackageAccessSimplaJava(
                                    Path.of(path1).toUri(),
                                    Kind.CLASS, path1, file);
                                result.add(withPackageAccessSimplaJava);
                            }
                            return FileVisitResult.CONTINUE;
                        }

                        @Override
                        public FileVisitResult visitFileFailed(Path file, IOException exc)
                            throws IOException {
                            return FileVisitResult.CONTINUE;
                        }

                        @Override
                        public FileVisitResult postVisitDirectory(Path dir, IOException exc)
                            throws IOException {
                            return FileVisitResult.CONTINUE;
                        }
                    });
                } catch (IOException e) {
                    e.printStackTrace();
                }
            });
        } else {
            var springbootjarpath = Arrays.stream(System.getProperty("java.class.path").split(";"))
                .filter(s -> s.endsWith(".jar"));
            var tempPath = Path.of(getPath()).getParent().resolve("temp");
            if (Files.notExists(tempPath)) {
                Files.createDirectories(tempPath);
            }
            springbootjarpath.forEach(s -> {
                try {
                    //直接解压

                    var jarFile = new JarFile(s);
                    jarFile.stream().filter(jarEntry -> jarEntry.getName().contains("BOOT-INF"))
                        .filter(jarEntry -> !jarEntry.isDirectory())
                        .forEach(jarEntry -> {
                            try {
                                if (Files
                                    .notExists(tempPath.resolve(jarEntry.getName()).getParent())) {
                                    Files.createDirectories(
                                        tempPath.resolve(jarEntry.getName()).getParent());
                                }

                                Files.deleteIfExists(tempPath.resolve(jarEntry.getName()));
                                var filePath = Files
                                    .createFile(tempPath.resolve(jarEntry.getName()));

                                var ou = Files.newOutputStream(filePath);
                                var in = jarFile.getInputStream(jarEntry);
                                IOUtils.copy(in, ou);
                                in.close();
                                ou.close();
                            } catch (IOException e) {
                                e.printStackTrace();
                            }
                        });

                } catch (IOException e) {
                    e.printStackTrace();
                }
            });

            Files.walk(tempPath.resolve("BOOT-INF"))
                .filter(f -> !Files.isDirectory(f))
                .forEach(s -> {
                    if (s.toString().endsWith(".jar")) {
                        try {
                            JarFile jarFile = new JarFile(s.toString());
                            jarFile
                                .stream()
                                .forEach(jarEntry -> {
                                    if (!jarEntry.isDirectory() && jarEntry.getName()
                                        .endsWith(".class")) {
                                        try (var in = jarFile.getInputStream(jarEntry)) {
                                            var jarPath = Path.of(s.toString());
                                            String separator =
                                                jarEntry.getName().startsWith("/") ? "!" : "!/";
                                            ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
                                            IOUtils.copy(in, outputStream);
                                            result.add(
                                                new NewJarJavaFileObject(
                                                    URI.create(
                                                        "jar:///" + jarPath.toUri().normalize()
                                                            + separator
                                                            + jarEntry
                                                            .getName()),
                                                    Kind.CLASS,
                                                    outputStream,
                                                    jarEntry
                                                ));
                                        } catch (IOException e) {
                                            e.printStackTrace();
                                        }
                                    }
                                });

                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    } else if (s.toString().endsWith(".class")) {
                        WithPackageAccessSimplaJava withPackageAccessSimplaJava = new WithPackageAccessSimplaJava(
                            s.toUri(),
                            Kind.CLASS,
                            tempPath.resolve("BOOT-INF").resolve("classes").normalize().toString(),
                            s);
                        result.add(withPackageAccessSimplaJava);
                    }
                });
        }
        return result;
    }

    public static String getPath() {
        ApplicationHome home = new ApplicationHome(DBClazz.class);
        return home.getSource().getPath();
    }

    /**
     * 判断是否jar模式运行
     */
    public static boolean isJar() {
        return StringUtils.isNotEmpty(System.getProperty("loader.path"));
    }


    public static void main(String[] args) {
        resolve();
    }
}