文章目录
Java 读取任意shapefile的所有字段，并插入到MongoDB数据库（Spring Boot）
- 亮点：动态获取字段信息，动态创建数据库表信息，支持任意shapefile数据
1. 统一返回结果封装
/**
* @program: tool
* @description:
* @author: Mr.superbeyone
* @create: 2018-10-18 16:37
**/
@Data
public class JsonResult<T> {
private Integer code;
private String msg;
private T data;
//使用 lombok @Data注解 或者 Get Set方法 ...
}
2. shp文件数据实体封装
/**
* @program: tool
* @description:
* @author: Mr.superbeyone
* @create: 2018-10-16 12:12
**/
@Data
public class ShapeModel implements Serializable {
private String id;
private String type;
private String geometry;
private String properties;
//使用 lombok @Data注解 或者 Get Set方法 ...
}
3. 核心代码
/**
* 读取shp文件
*
* @param filePath 读取的文件路径
* @param collection mongoDB collection
* @return
*/
private JsonResult<Set<String>> readShapeFile(String filePath, MongoCollection<Document> collection) {
JsonResult<Set<String>> result = new JsonResult<>();
Set<String> set = new HashSet<>();
File folder = new File(filePath);
if (!folder.isDirectory()) {
if (folder.toString().endsWith(".shp")) {
try {
return getShapeFile(folder, collection);
} catch (Exception e) {
e.printStackTrace();
}
} else {
System.out.println("选择的文件后缀名不是.shp");
return null;
}
} else {
File[] files = folder.listFiles();
if (files.length <= 0) {
System.out.println("目录文件为空");
return null;
}
for (File file : files) {
if (!file.toString().endsWith(".shp")) {
continue;
}
try {
result = getShapeFile(file, collection);
Set fields = result.getData();
set.addAll(fields);
} catch (Exception e) {
e.printStackTrace();
}
}
result.setData(set);
}
return result;
}
private JsonResult<Set<String>> getShapeFile(File file, MongoCollection<Document> collection) {
JsonResult<Set<String>> result = new JsonResult();
HashSet<String> fields = new HashSet<>();
int number = 0;
int index = 0;
try {
ShapefileDataStore shapefileDataStore = new ShapefileDataStore(file.toURI().toURL());
shapefileDataStore.setCharset(Charset.forName("utf-8"));
ContentFeatureSource featureSource = shapefileDataStore.getFeatureSource();
SimpleFeatureIterator featureIterator = featureSource.getFeatures().features();
ArrayList<Document> documents = new ArrayList<>();
Document document = null;
while (featureIterator.hasNext()) {
number++;
SimpleFeature feature = featureIterator.next();
Collection<? extends Property> properties = feature.getValue();
//feature转GeoJSON
FeatureJSON featureJSON = new FeatureJSON();
StringWriter writer = new StringWriter();
featureJSON.writeFeature(feature, writer);
//TODO: 判断writer.toString().length() 否则可能会抛下列异常
//org.bson.BsonSerializationException: Payload document size of is larger than maximum of 16777216
ShapeModel shapeModel = JSON.parseObject(writer.toString(), ShapeModel.class);
document = new Document();
if (properties.size() < 0) {
continue;
}
if (index == 0) {
for (Property property : properties) {
if (property.getType().getClass() == org.geotools.feature.type.AttributeTypeImpl.class) {
fields.add(property.getName().toString());
} else {
//空间字段
fields.add("#" + property.getName() + "#");
}
}
result.setData(fields);
result.setCode(ResultCodeEnum.GET_UPDATE_FILE_FIELDS_SUCCESS.getCode());
result.setMsg(ResultCodeEnum.GET_UPDATE_FILE_FIELDS_SUCCESS.getMsg());
//TODO:创建分片数据库
//createShardDatabase(fields);
}
for (Property property : properties) {
if (property.getType().getClass() == org.geotools.feature.type.AttributeTypeImpl.class) {
if (property.getValue() == null) {
document.put(property.getName().toString(), "");
} else {
document.put(property.getName().toString(), property.getValue().toString());
}
} else {
//空间字段
document.put(property.getName().toString(), shapeModel.getGeometry());
}
}
index++;
//逐条插入
//collection.insertOne(document);
documents.add(document);
//每1000条数据插入一次
if (number % 1000 == 0) {
collection.insertMany(documents);
documents = new ArrayList<>();
}
//最后插入不足1000条的数据
if (!featureIterator.hasNext()) {
collection.insertMany(documents);
}
}
} catch (Exception e) {
//出现异常
result.setMsg("插入第" + (index + 1) + "条时出现错误");
result.setCode(4000);
}
return result;
}
}
3. 可能出现的异常
org.bson.BsonSerializationException: Payload document size of is larger than maximum of 16777216
3.1 异常产生原因
3.2 解决方案
- 判断所要插入的数据的大小，如果大于16M，可以选择跳过，或者停止插入
4 扩展
4.1 创建数据库分片
public void createShardDatabase(Set<String> fields) {
//TODO:使用 databaseName 生成策略
String databaseName = "super";
//TODO:使用 collectionName 生成策略
String collectionName = "beyone";
List<String> set = new ArrayList<>();
Map<String, Object> map = new HashMap<String, Object>();
if (fields.size() < 0) {
return;
}
for (String field : fields) {
//空间字段
if (field.startsWith("#") && field.endsWith("#")) {
String splitField = field.split("#")[1];
map.put(splitField, "hashed");
} else {
set.add(field);
}
}
//mongoDBConfig 是通过注入获得的
/**
* @Autowired
* MongoDBConfig mongoDBConfig;
*/
MongoClient client = new MongoClient(mongoDBConfig.getHost(), mongoDBConfig.getPort());
MongoDatabase adminDB = client.getDatabase(mongoDBConfig.getDatabase());
Document enableShardDoc = new Document();
enableShardDoc.put("enablesharding", databaseName);
Document result = adminDB.runCommand(enableShardDoc);
Object obj = result.get("ok");
if (null == obj) {
}
if ("1.0".equals(obj.toString())) {
Document shardKeyDoc = new Document();
String coll = databaseName + "." + collectionName;
shardKeyDoc.put("shardcollection", coll);
Document t = new Document();
t.putAll(map);
shardKeyDoc.put("key", t);
result = adminDB.runCommand(shardKeyDoc);
obj = result.get("ok");
if (null == obj) {
//logger.info("error");
}
if ("1.0".equals(obj.toString())) {
//logger.info("success");
} else {
//logger.info("error");
}
} else {
//logger.info("error");
}
}
