如何构建回测系统（十）
问题
在（九）中实现了一个可以从数据库中编译并加载代码的classloader，但这是有缺陷的
缺陷在于，如果要使用回测系统实现的类和对象则会出现编译无法通过
提示Symbol找不到或者缺失
定位问题
日常我们使用编译功能一般都是maven或者是ant等框架来编译，很少说直接用javac来直接对整个项目编译
在（九）使用的compiler其实就是javac的java代码版本如果采用javac编译一个引用了其他java或者class的文件，则会出现很类似的症状
找到Maven/IDE等工具具体执行javac的代码其实就能看到，是编译时候带上了参数-cp/-classpath/
解决问题
compiler的getTask函数中有个参数叫options，根据文档，这是类似于javac中的参数，但其实如果我们简单的把依赖的class按照-cp的形式添加到参数中就会发现直接报错
这是因为-cp是传给jvm的参数，getTask函数只支持具体的54个参数，属于编译期间具体的信息数据用的
而-cp产生作用的方式是在FileManager中的list()函数起作用的（这里完全没有任何文档，只能单步调试代码发现，所以这里无法说以后就一定正确）
我们只需要将依赖的class/jar/java文件按照需求变更成SimpleJavaFileObject既可
参考代码
因为环境因素，这里的参考代码是在springboot中实现的
package com.michaelssss.domain.strategy;
import static javax.tools.StandardLocation.CLASS_PATH;
import com.michaelssss.exception.RunException;
import com.michaelssss.utils.ArrayUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.tools.DiagnosticCollector;
import javax.tools.FileObject;
import javax.tools.ForwardingJavaFileManager;
import javax.tools.JavaCompiler;
import javax.tools.JavaFileManager;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import javax.tools.SimpleJavaFileObject;
import javax.tools.ToolProvider;
import org.springframework.stereotype.Component;
@Component
public class MyCompiler {
public byte[] compile(String name, String codes) {
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
var diagnosticCollector = new DiagnosticCollector<JavaFileObject>();
var fileManager = new MyJavaFileObjectManager(
compiler.getStandardFileManager(diagnosticCollector, Locale.CHINESE,
StandardCharsets.UTF_8));
fileManager.complieName = name;
var javaFileObjects = Collections
.singletonList(
new MyJavaFileObject(name.replace('.', '/'), codes, fileManager.outputStream));
var options = new ArrayList<String>();
var compilerTask = compiler
.getTask(null, fileManager, diagnosticCollector, options, null, javaFileObjects);
var result = compilerTask.call();
if (Objects.nonNull(result) && result) {
return fileManager.outputStream.toByteArray();
} else {
StringBuilder stringBuilder = new StringBuilder();
diagnosticCollector.getDiagnostics()
.forEach(diagnostic -> stringBuilder.append(
diagnostic.toString()
).append("\r\n"));
throw new RunException(stringBuilder.toString());
}
}
static class MyJavaFileObjectManager extends ForwardingJavaFileManager<JavaFileManager> {
Map<String, JavaFileObject> fileObjects = new HashMap<>();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
List<JavaFileObject> result;
String complieName;
public MyJavaFileObjectManager(JavaFileManager fileManager) {
super(fileManager);
}
@Override
public JavaFileObject getJavaFileForInput(Location location, String className,
Kind kind) throws IOException {
JavaFileObject javaFileObject = fileObjects.get(className);
if (!location.getName().contains(CLASS_PATH.getName())) {
return super.getJavaFileForInput(location, className, kind);
}
if (Objects.isNull(javaFileObject)) {
if (Objects.isNull(result)) {
result = CompilerDependenceResolver.resolve();
}
var aa = result.stream().filter(
f -> f.getName().equalsIgnoreCase(className) && kind.equals(f.getKind()))
.collect(
Collectors.toList());
if (aa.isEmpty()) {
return super.getJavaFileForInput(location, className, kind);
} else {
return aa.get(0);
}
}
return javaFileObject;
}
@Override
public JavaFileObject getJavaFileForOutput(Location location, String qualifiedClassName,
JavaFileObject.Kind kind, FileObject sibling) throws IOException {
JavaFileObject javaFileObject = new MyJavaFileObject(qualifiedClassName, kind,
outputStream);
fileObjects.put(qualifiedClassName, javaFileObject);
return javaFileObject;
}
@Override
public String inferBinaryName(Location location, JavaFileObject file) {
if (file instanceof MyJavaFileObject) {
return file.getName();
}
if (file instanceof WithPackageAccessSimplaJava) {
return ((WithPackageAccessSimplaJava) file).getBinaryName();
}
if (
file instanceof NewJarJavaFileObject
) {
return ((NewJarJavaFileObject) file).getBinaryName();
} else {
return super.inferBinaryName(location, file);
}
}
@Override
public Iterable<JavaFileObject> list(Location location, String packageName, Set<Kind> kinds,
boolean recurse) throws IOException {
if (location.getName().contains(CLASS_PATH.getName())) {
if (Objects.isNull(result)) {
result = CompilerDependenceResolver.resolve();
}
if (!result.isEmpty()) {
var a = result.stream()
.filter(
r -> ArrayUtils
.isContain(((MySimpleJavaObjectI) r).getBinaryName().split("\\."),
packageName.split("\\.")) && kinds
.contains(r.getKind()))
.collect(Collectors.toList());
var e = super.list(location, packageName, kinds, recurse);
e.forEach(a::add);
return a;
}
}
return super.list(location, packageName, kinds, recurse);
}
}
static class MyJavaFileObject extends SimpleJavaFileObject {
String source;
OutputStream outputStream;
String name;
public MyJavaFileObject(String name, Kind kind, OutputStream stream) {
super(URI.create("String:///" + name + kind.extension), kind);
this.outputStream = stream;
}
public MyJavaFileObject(String name, String source) {
super(URI.create("String:///" + name + Kind.SOURCE.extension), Kind.SOURCE);
this.source = source;
this.name = name;
}
public MyJavaFileObject(String name, String source, OutputStream outputStream) {
this(name, source);
this.outputStream = outputStream;
}
public MyJavaFileObject(URI uri, Kind kind) {
super(uri, kind);
}
@Override
public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {
return source;
}
@Override
public OutputStream openOutputStream() throws IOException {
return outputStream;
}
}
}
package com.michaelssss.domain.strategy;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import javax.tools.JavaFileObject;
import javax.tools.JavaFileObject.Kind;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.boot.system.ApplicationHome;
public class CompilerDependenceResolver {
/**
* @return 返回所有非SystemLoader加载的class的JavaFileObject
*/
public static List<String> getClassPath() {
var systemType = System.getProperty("os.name");
var classpath = new String[0];
if (systemType.contains("Windows")) {
classpath = System.getProperty("java.class.path").split(";");
}
if (systemType.contains("Linux")) {
classpath = System.getProperty("java.class.path").split(":");
}
return Arrays.asList(classpath);
}
@SneakyThrows
public static List<JavaFileObject> resolve() {
List<JavaFileObject> result = new ArrayList<>();
//查询所有非springboot主文件本身的jar
final List<String> jarlibs = new ArrayList<>();
var loadPath = System.getProperty("loader.path");
var classpath = System.getProperty("java.class.path").split(";");
if (StringUtils.isNotEmpty(loadPath)) {
classpath = Files.list(Path.of(loadPath))
.filter(path -> !Files.isDirectory(path) && path.toString().endsWith(".jar"))
.map(Path::toString).toArray(String[]::new);
}
jarlibs.addAll(Arrays.stream(classpath).filter(s -> s.endsWith(".jar"))
.collect(Collectors.toList()));
var compilerModuleClassPaths = Arrays.stream(classpath).filter(s -> !s.endsWith(".jar"))
.collect(Collectors.toList());
if (jarlibs.isEmpty()) {
throw new Error("load jar info failed");
}
jarlibs.forEach(s -> {
try {
JarFile jarFile = new JarFile(s);
jarFile
.stream()
.forEach(jarEntry -> {
if (!jarEntry.isDirectory() && jarEntry.getName().endsWith(".class")) {
try (var in = jarFile.getInputStream(jarEntry)) {
var jarPath = Path.of(s);
String separator = jarEntry.getName().startsWith("/") ? "!" : "!/";
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
IOUtils.copy(in, outputStream);
result.add(
new NewJarJavaFileObject(
URI.create(
"jar:///" + jarPath.toUri().normalize() + separator
+ jarEntry
.getName()),
Kind.CLASS,
outputStream,
jarEntry
));
} catch (IOException e) {
e.printStackTrace();
}
}
});
} catch (IOException e) {
e.printStackTrace();
}
});
if (!isJar()) {
var path = Path.of(getPath());
Files.walkFileTree(path, new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
if (file.toString().endsWith(".class")) {
WithPackageAccessSimplaJava withPackageAccessSimplaJava = new WithPackageAccessSimplaJava(
path.toUri(),
Kind.CLASS, path.normalize().toString(), file);
result.add(withPackageAccessSimplaJava);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc)
throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc)
throws IOException {
return FileVisitResult.CONTINUE;
}
});
compilerModuleClassPaths.forEach(path1 -> {
try {
Files.walkFileTree(Path.of(path1), new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs)
throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
if (file.toString().endsWith(".class")) {
WithPackageAccessSimplaJava withPackageAccessSimplaJava = new WithPackageAccessSimplaJava(
Path.of(path1).toUri(),
Kind.CLASS, path1, file);
result.add(withPackageAccessSimplaJava);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc)
throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc)
throws IOException {
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
e.printStackTrace();
}
});
} else {
var springbootjarpath = Arrays.stream(System.getProperty("java.class.path").split(";"))
.filter(s -> s.endsWith(".jar"));
var tempPath = Path.of(getPath()).getParent().resolve("temp");
if (Files.notExists(tempPath)) {
Files.createDirectories(tempPath);
}
springbootjarpath.forEach(s -> {
try {
//直接解压
var jarFile = new JarFile(s);
jarFile.stream().filter(jarEntry -> jarEntry.getName().contains("BOOT-INF"))
.filter(jarEntry -> !jarEntry.isDirectory())
.forEach(jarEntry -> {
try {
if (Files
.notExists(tempPath.resolve(jarEntry.getName()).getParent())) {
Files.createDirectories(
tempPath.resolve(jarEntry.getName()).getParent());
}
Files.deleteIfExists(tempPath.resolve(jarEntry.getName()));
var filePath = Files
.createFile(tempPath.resolve(jarEntry.getName()));
var ou = Files.newOutputStream(filePath);
var in = jarFile.getInputStream(jarEntry);
IOUtils.copy(in, ou);
in.close();
ou.close();
} catch (IOException e) {
e.printStackTrace();
}
});
} catch (IOException e) {
e.printStackTrace();
}
});
Files.walk(tempPath.resolve("BOOT-INF"))
.filter(f -> !Files.isDirectory(f))
.forEach(s -> {
if (s.toString().endsWith(".jar")) {
try {
JarFile jarFile = new JarFile(s.toString());
jarFile
.stream()
.forEach(jarEntry -> {
if (!jarEntry.isDirectory() && jarEntry.getName()
.endsWith(".class")) {
try (var in = jarFile.getInputStream(jarEntry)) {
var jarPath = Path.of(s.toString());
String separator =
jarEntry.getName().startsWith("/") ? "!" : "!/";
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
IOUtils.copy(in, outputStream);
result.add(
new NewJarJavaFileObject(
URI.create(
"jar:///" + jarPath.toUri().normalize()
+ separator
+ jarEntry
.getName()),
Kind.CLASS,
outputStream,
jarEntry
));
} catch (IOException e) {
e.printStackTrace();
}
}
});
} catch (IOException e) {
e.printStackTrace();
}
} else if (s.toString().endsWith(".class")) {
WithPackageAccessSimplaJava withPackageAccessSimplaJava = new WithPackageAccessSimplaJava(
s.toUri(),
Kind.CLASS,
tempPath.resolve("BOOT-INF").resolve("classes").normalize().toString(),
s);
result.add(withPackageAccessSimplaJava);
}
});
}
return result;
}
public static String getPath() {
ApplicationHome home = new ApplicationHome(DBClazz.class);
return home.getSource().getPath();
}
/**
* 判断是否jar模式运行
*/
public static boolean isJar() {
return StringUtils.isNotEmpty(System.getProperty("loader.path"));
}
public static void main(String[] args) {
resolve();
}
}