【hadoop source】hadoop配置信息处理

露亚
320 阅读1分钟

UML/简介

conf.png

  • Configuration：负责XML配置文件、配置项kv的相关操作
    • 加载：加载配置文件到内存
    • 解析：解析XML语法获取配置项kv
    • 合并：多个配置文件合并，相同的配置项覆盖
    • 扩展：如${hadoop.tmp.dir}/tmp格式路径，转化为对应的实际路径
    • 存储：配置文件到resources，配置项到properties等
  • Resource：Configuration的内部类，封装XML配置文件，类型有：Path、File、InputStream、字符串url
  • Configurable：实现类可绑定Configuration实例，获取其中的配置项v
  • Configured：Configurable Base（基础实现类）
  • Reconfigurable：实现类可修改运行时可变配置项
  • Writable：实现类可进行hadoop序列化

流程

配置文件读取

  1. 对于conf目录下的文件，调用addResource()添加XML配置文件到resources（XML配置文件列表）中，static块中调用addDefaultResource()添加默认XML配置文件到defaultResources中
  2. 调用reloadConfiguration()重新加载配置项（lazy，实际是清空properties和final配置项k，等到调用get()时再用loadResources方法加载）
  3. loadResources()从XML配置文件列表resources和defaultResources加载所有配置项到properties：调用loadResource()使用dom解析XML配置文件，得到配置项写入properties，并将解析后的properties更新到resources；调用subsituteVars()使用正则匹配${...}并使用System类获取环境变量进行属性扩展

配置实例使用

  1. 反射创建Configurable实现类实例
  2. Configurable实现类实例调用setConf()设置Configuration实例。之后即可调用getConf()获取Configuration实例，设置/获取配置项

核心代码（最后的main test，可直接运行）

测试文件（resources文件夹下）

  • core-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>build/test</value>
        <description>A base for other temporary directories.</description>
    </property>

    <property>
        <name>site</name>
        <value>${hadoop.tmp.dir}/s3.name</value>    <!-- 扩展 -->
        <description>The name of the s3 file system for testing.</description>
        <final>true</final>
    </property>
</configuration>
  • core-default.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>build/test</value>
        <description>A base for other temporary directories.</description>
    </property>

    <property>
        <name>default</name>
        <value>s3:///</value>
        <description>The name of the s3 file system for testing.</description>
    </property>
</configuration>

配置文件读写

  • Configuration
import org.w3c.dom.*;
import java.util.*;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.net.URL;
import java.nio.file.Path;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Configuration {

    private ArrayList<Resource> resources = new ArrayList<>();
    private boolean loadDefaults = true;
    private static final WeakHashMap<Configuration, Object> REGISTRY = new WeakHashMap<>(); //保存系统Configuration实例，如HBaseConfiguration
    private static final CopyOnWriteArrayList<String> defaultResources = new CopyOnWriteArrayList<>();
    /**
     * 添加默认XML配置文件
     */
    static{
        ClassLoader cl = Thread.currentThread().getContextClassLoader();
        if (cl == null){
            cl = Configuration.class.getClassLoader();
        }
        addDefaultResource("core-default.xml");
    }
    private Set<String> finalParameters = new HashSet<>();  //final配置项k，防止合并时被覆盖
    private Properties properties;
    private Properties overlay; //set配置项
    private ClassLoader classLoader;    //加载配置文件
    {
        classLoader = Thread.currentThread().getContextClassLoader();
        if (classLoader == null){
            classLoader = Configuration.class.getClassLoader();
        }
    }



    public Configuration(){
        this.loadDefaults = true;   //可作为参数
        synchronized (Configuration.class){
            REGISTRY.put(this, null);
        }
    }



    /**
     * 封装XML配置文件
     */
    private static class Resource{
        private final Object resource;
        private final String name;

        public Resource(Object resource){
            this(resource, resource.toString());
        }
        public Resource(Object resource, String name){
            this.resource = resource;
            this.name = name;
        }

        public Object getResource() {
            return resource;
        }
        public String getName() {
            return name;
        }
        @Override
        public String toString() {
            return name;
        }
    }
    /**
     * 添加XML配置文件
     */
    public void addResource(String name){
        addResourceObject(new Resource(name));
    }
    public void addResource(Path path){     //暂不考虑
        addResourceObject(new Resource(path));
    }
    private void addResourceObject(Resource resource){
        resources.add(resource);
        reloadConfiguration();
    }
    /**
     * 重新加载配置项（清空，get()时再lazy加载）
     */
    private synchronized void reloadConfiguration(){
        properties = null;
        finalParameters.clear();
    }
    /**
     * 添加默认XML配置文件
     * 其他所有需要加载默认XML配置文件的Configuration实例重新加载配置项
     */
    public static synchronized void addDefaultResource(String name){
        if(! defaultResources.contains(name)){
            defaultResources.add(name);
            for (Configuration conf : REGISTRY.keySet()){   //其他Configuration实例重新加载配置项
                if (conf.loadDefaults){
                    conf.reloadConfiguration();
                }
            }
        }
    }



    /**
     * 获取配置项
     * 暂不考虑弃用属性
     */
    public String get(String name){
        return substituteVars(getProps().getProperty(name));
    }
    /**
     * 获取properties实例
     * 暂不考虑记录<加载/修改的name，Resource>
     */
    protected synchronized Properties getProps(){
        if (properties == null) {
            properties = new Properties();
            loadResources(properties, resources);   //加载
            if (overlay != null) {
                properties.putAll(overlay); //包含关系
            }
        }
        return properties;
    }
    /**
     * 加载配置项到properties
     */
    private void loadResources(Properties properties, ArrayList<Resource> resources){
        if (loadDefaults){
            for (String resource : defaultResources){
                loadResource(properties, new Resource(resource));
            }
        }
        for (int i=0; i< resources.size(); i++){
            resources.set(i, loadResource(properties, resources.get(i)));
        }
    }
    /**
     * DOM解析XML配置文件
     * 更新resources中的Resource，Resource(Object, name) -> Resource(Properties, name)
     * 只解析String类型（文件名），暂不考虑其他类型以及各种非空判断
     */
    private Resource loadResource(Properties properties, Resource wrapper){
        try{
            Object resource = wrapper.getResource();
            String name = wrapper.getName();

            DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
            docBuilderFactory.setIgnoringComments(true);
            docBuilderFactory.setNamespaceAware(true);
            docBuilderFactory.setXIncludeAware(true);

            DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
            Document doc = null;
            Element root;
            if (resource instanceof String){
                URL url = classLoader.getResource((String) resource);   //加载
                doc = builder.parse(url.openStream(), url.toString());  //解析
            }
            if (doc == null){
                throw new RuntimeException("parse failed...");
            }
            //获取内容
            root = doc.getDocumentElement();
            Properties toAddTo = properties;
            NodeList props = root.getChildNodes();
            for (int i=0; i<props.getLength(); i++){
                Node propNode = props.item(i);
                if (! (propNode instanceof Element)){
                    continue;
                }
                Element prop = (Element) propNode;
                NodeList fields = prop.getChildNodes();
                String attr = null;
                String value = null;
                boolean finalParameter = false;
                for (int j=0; j<fields.getLength(); j++) {
                    Node fieldNode = fields.item(j);
                    if (! (fieldNode instanceof Element))
                        continue;
                    Element field = (Element)fieldNode;
                    if ("name".equals(field.getTagName()) && field.hasChildNodes())
                        attr = ((Text)field.getFirstChild()).getData().trim();
                    if ("value".equals(field.getTagName()) && field.hasChildNodes())
                        value = ((Text)field.getFirstChild()).getData();
                    if ("final".equals(field.getTagName()) && field.hasChildNodes())
                        finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
                }
                loadProperty(toAddTo, attr, value, finalParameter); //添加配置项到toAddTo，暂不考虑final配置项合并覆盖
            }
            overlay(properties, toAddTo);   //复制
            return new Resource(toAddTo, name); //更新resources
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    private void loadProperty(Properties properties, String attr, String value, boolean finalParameter) {
        if (! finalParameters.contains(attr)) {
            properties.setProperty(attr, value);
        }
        if (finalParameter) {
            finalParameters.add(attr);
        }
    }
    /**
     * Properties复制
     */
    private void overlay(Properties to, Properties from){
        for (Map.Entry<Object, Object> entry : from.entrySet()){
            to.put(entry.getKey(), entry.getValue());
        }
    }
    /**
     * 扩展（正则匹配）
     */
    private static Pattern varPat = Pattern.compile("\$\{[^\}\$\u0020]+\}");
    private static int MAX_SUBST = 20;
    private String substituteVars(String expr) {
        Matcher match = varPat.matcher("");
        String eval = expr;
        for(int s=0; s<MAX_SUBST; s++) {
            match.reset(eval);
            if (! match.find()) {
                return eval;
            }
            String var = match.group();
            var = var.substring(2, var.length()-1); //去除${}
            String val = null;
            try {
                val = System.getProperty(var);
            } catch(SecurityException ignored) {

            }
            if (val == null) {
                val = getRaw(var);
            }
            if (val == null) {
                return eval;    //${var}未绑定，原样返回
            }
            eval = eval.substring(0, match.start()) + val + eval.substring(match.end());
        }
        return null;
    }
    private String getRaw(String name){
        return getProps().getProperty(name);
    }



    /**
     * 设置配置项
     */
    public void set(String name, String value){
        getOverlay().setProperty(name, value);
        getProps().setProperty(name, value);
    }
    private synchronized Properties getOverlay() {
        if (overlay==null){
            overlay=new Properties();
        }
        return overlay;
    }



    /**
     * main test
     */
    public static void main(String[] args) {
        Configuration conf = new Configuration();   //加载默认XML配置文件

        System.out.println(defaultResources);

        conf.addResource("core-site.xml");
        System.out.println(conf.resources);

        System.out.println(conf.properties);    //lazy
        System.out.println(conf.get("site"));
        System.out.println(conf.properties);

        conf.set("hadoop.tmp.dir", "set/test");
        System.out.println(conf.get("hadoop.tmp.dir"));
    }
}

配置实例使用（对外提供接口，简单实现）

  • Configurable
public interface Configurable {

    void setConf(Configuration conf);

    Configuration getConf();
}
  • Configured
public class Configured implements Configurable{

    Configuration conf;
    public Configured(){
        this(null);
    }
    public Configured(Configuration conf){
        setConf(conf);
    }
    @Override
    public void setConf(Configuration conf) {
        this.conf = conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }



    public static void main(String[] args) {
        Configuration conf = new Configuration();   //配置文件应在反射创建对象时给出
        conf.addResource("core-site.xml");

        Configured cfed = new Configured(conf);
        System.out.println(cfed.getConf().get("hadoop.tmp.dir"));
    }
}
avatar
文章
阅读
粉丝