path: root/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.drill.exec.compile;

import java.io.IOException;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;

import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.compile.MergeAdapter.MergedClassResult;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.TypeValidators.EnumeratedStringValidator;
import org.codehaus.commons.compiler.CompileException;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.tree.ClassNode;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

/**
 * Compiles generated code, merges the resulting class with the
 * template class, and performs byte-code cleanup on the resulting
 * byte codes. The most important transform is scalar replacement
 * which replaces occurences of non-escaping objects with a
 * collection of member variables.
 */

public class ClassTransformer {
  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassTransformer.class);

  private static final int MAX_SCALAR_REPLACE_CODE_SIZE = 2*1024*1024; // 2meg

  private final ByteCodeLoader byteCodeLoader = new ByteCodeLoader();
  private final DrillConfig config;
  private final OptionManager optionManager;

  public final static String SCALAR_REPLACEMENT_OPTION =
      "org.apache.drill.exec.compile.ClassTransformer.scalar_replacement";
  public final static EnumeratedStringValidator SCALAR_REPLACEMENT_VALIDATOR = new EnumeratedStringValidator(
      SCALAR_REPLACEMENT_OPTION, "try", "off", "try", "on");

  @VisibleForTesting // although we need it even if it weren't used in testing
  public enum ScalarReplacementOption {
    OFF, // scalar replacement will not ever be used
    TRY, // scalar replacement will be attempted, and if there is an error, we fall back to not using it
    ON; // scalar replacement will always be used, and any errors cause user visible errors

    /**
     * Convert a string to an enum value.
     *
     * @param s the string
     * @return an enum value
     * @throws IllegalArgumentException if the string doesn't match any of the enum values
     */
    public static ScalarReplacementOption fromString(final String s) {
      switch(s) {
      case "off":
        return OFF;
      case "try":
        return TRY;
      case "on":
        return ON;
      default:
        throw new IllegalArgumentException("Invalid ScalarReplacementOption \"" + s + "\"");
      }
    }
  }

  public ClassTransformer(final DrillConfig config, final OptionManager optionManager) {
    this.config = config;
    this.optionManager = optionManager;
  }

  public static class ClassSet {
    public final ClassSet parent;
    public final ClassNames precompiled;
    public final ClassNames generated;

    public ClassSet(ClassSet parent, String precompiled, String generated) {
      Preconditions.checkArgument(!generated.startsWith(precompiled),
          String.format("The new name of a class cannot start with the old name of a class, otherwise class renaming will cause problems. Precompiled class name %s. Generated class name %s",
              precompiled, generated));
      this.parent = parent;
      this.precompiled = new ClassNames(precompiled);
      this.generated = new ClassNames(generated);
    }

    public ClassSet getChild(String precompiled, String generated) {
      return new ClassSet(this, precompiled, generated);
    }

    public ClassSet getChild(String precompiled) {
      return new ClassSet(this, precompiled, precompiled.replace(this.precompiled.dot, this.generated.dot));
    }

    @Override
    public int hashCode() {
      final int prime = 31;
      int result = 1;
      result = prime * result + ((generated == null) ? 0 : generated.hashCode());
      result = prime * result + ((parent == null) ? 0 : parent.hashCode());
      result = prime * result + ((precompiled == null) ? 0 : precompiled.hashCode());
      return result;
    }

    @Override
    public boolean equals(Object obj) {
      if (this == obj) {
        return true;
      }
      if (obj == null) {
        return false;
      }
      if (getClass() != obj.getClass()) {
        return false;
      }
      ClassSet other = (ClassSet) obj;
      if (generated == null) {
        if (other.generated != null) {
          return false;
        }
      } else if (!generated.equals(other.generated)) {
        return false;
      }
      if (parent == null) {
        if (other.parent != null) {
          return false;
        }
      } else if (!parent.equals(other.parent)) {
        return false;
      }
      if (precompiled == null) {
        if (other.precompiled != null) {
          return false;
        }
      } else if (!precompiled.equals(other.precompiled)) {
        return false;
      }
      return true;
    }
  }

  public static class ClassNames {
    public final String dot;
    public final String slash;
    public final String clazz;

    public ClassNames(String className) {
      dot = className;
      slash = className.replace('.', FileUtils.separatorChar);
      clazz = FileUtils.separatorChar + slash + ".class";
    }

    @Override
    public int hashCode() {
      final int prime = 31;
      int result = 1;
      result = prime * result + ((clazz == null) ? 0 : clazz.hashCode());
      result = prime * result + ((dot == null) ? 0 : dot.hashCode());
      result = prime * result + ((slash == null) ? 0 : slash.hashCode());
      return result;
    }

    @Override
    public boolean equals(Object obj) {
      if (this == obj) {
        return true;
      }
      if (obj == null) {
        return false;
      }
      if (getClass() != obj.getClass()) {
        return false;
      }
      ClassNames other = (ClassNames) obj;
      if (clazz == null) {
        if (other.clazz != null) {
          return false;
        }
      } else if (!clazz.equals(other.clazz)) {
        return false;
      }
      if (dot == null) {
        if (other.dot != null) {
          return false;
        }
      } else if (!dot.equals(other.dot)) {
        return false;
      }
      if (slash == null) {
        if (other.slash != null) {
          return false;
        }
      } else if (!slash.equals(other.slash)) {
        return false;
      }
      return true;
    }
  }

  public Class<?> getImplementationClass(CodeGenerator<?> cg) throws ClassTransformationException {
    final QueryClassLoader loader = new QueryClassLoader(config, optionManager);
    return getImplementationClass(loader, cg.getDefinition(),
        cg.getGeneratedCode(), cg.getMaterializedClassName());
  }

  public Class<?> getImplementationClass(
      final QueryClassLoader classLoader,
      final TemplateClassDefinition<?> templateDefinition,
      final String entireClass,
      final String materializedClassName) throws ClassTransformationException {
    // unfortunately, this hasn't been set up at construction time, so we have to do it here
    final ScalarReplacementOption scalarReplacementOption = ScalarReplacementOption.fromString(optionManager.getOption(SCALAR_REPLACEMENT_VALIDATOR));

    try {
      final long t1 = System.nanoTime();
      final ClassSet set = new ClassSet(null, templateDefinition.getTemplateClassName(), materializedClassName);
      final byte[][] implementationClasses = classLoader.getClassByteCode(set.generated, entireClass);

      long totalBytecodeSize = 0;
      Map<String, ClassNode> classesToMerge = Maps.newHashMap();
      for (byte[] clazz : implementationClasses) {
        totalBytecodeSize += clazz.length;
        final ClassNode node = AsmUtil.classFromBytes(clazz, ClassReader.EXPAND_FRAMES);
        if (!AsmUtil.isClassOk(logger, "implementationClasses", node)) {
          throw new IllegalStateException("Problem found with implementationClasses");
        }
        classesToMerge.put(node.name, node);
      }

      final LinkedList<ClassSet> names = Lists.newLinkedList();
      final Set<ClassSet> namesCompleted = Sets.newHashSet();
      names.add(set);

      while ( !names.isEmpty() ) {
        final ClassSet nextSet = names.removeFirst();
        if (namesCompleted.contains(nextSet)) {
          continue;
        }
        final ClassNames nextPrecompiled = nextSet.precompiled;
        final byte[] precompiledBytes = byteCodeLoader.getClassByteCodeFromPath(nextPrecompiled.clazz);
        final ClassNames nextGenerated = nextSet.generated;
        final ClassNode generatedNode = classesToMerge.get(nextGenerated.slash);

        /*
         * TODO
         * We're having a problem with some cases of scalar replacement, but we want to get
         * the code in so it doesn't rot anymore.
         *
         *  Here, we use the specified replacement option. The loop will allow us to retry if
         *  we're using TRY.
         */
        MergedClassResult result = null;
        boolean scalarReplace = scalarReplacementOption != ScalarReplacementOption.OFF && entireClass.length() < MAX_SCALAR_REPLACE_CODE_SIZE;
        while(true) {
          try {
            result = MergeAdapter.getMergedClass(nextSet, precompiledBytes, generatedNode, scalarReplace);
            break;
          } catch(RuntimeException e) {
            // if we had a problem without using scalar replacement, then rethrow
            if (!scalarReplace) {
              throw e;
            }

            // if we did try to use scalar replacement, decide if we need to retry or not
            if (scalarReplacementOption == ScalarReplacementOption.ON) {
              // option is forced on, so this is a hard error
              throw e;
            }

            /*
             * We tried to use scalar replacement, with the option to fall back to not using it.
             * Log this failure before trying again without scalar replacement.
             */
            logger.info("scalar replacement failure (retrying)\n", e);
            scalarReplace = false;
          }
        }

        for (String s : result.innerClasses) {
          s = s.replace(FileUtils.separatorChar, '.');
          names.add(nextSet.getChild(s));
        }
        classLoader.injectByteCode(nextGenerated.dot, result.bytes);
        namesCompleted.add(nextSet);
      }

      Class<?> c = classLoader.findClass(set.generated.dot);
      if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
        logger.debug("Done compiling (bytecode size={}, time:{} millis).", DrillStringUtils.readable(totalBytecodeSize), (System.nanoTime() - t1) / 1000000);
        return c;
      }

      throw new ClassTransformationException("The requested class did not implement the expected interface.");
    } catch (CompileException | IOException | ClassNotFoundException e) {
      throw new ClassTransformationException(String.format("Failure generating transformation classes for value: \n %s", entireClass), e);
    }
  }
}