建设国家游戏网站,网站建设规范方案,个人养老保险,北京app搭建一、多表关联
输入是两个文件#xff0c;一个代表工厂表#xff0c;包含工厂名列和地址编号列#xff1b;另一个代表地址表#xff0c;包含地址名列和地址编号列。要求从输入数据中找出工厂名和地址名的对应关系#xff0c;输出工厂名——地址名表 二、maven…一、多表关联
输入是两个文件一个代表工厂表包含工厂名列和地址编号列另一个代表地址表包含地址名列和地址编号列。要求从输入数据中找出工厂名和地址名的对应关系输出工厂名——地址名表 二、maven设置
?xml version1.0 encodingUTF-8?project xmlnshttp://maven.apache.org/POM/4.0.0 xmlns:xsihttp://www.w3.org/2001/XMLSchema-instancexsi:schemaLocationhttp://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsdmodelVersion4.0.0/modelVersiongroupIdcom.mk/groupIdartifactIdspark-test/artifactIdversion1.0/versionnamespark-test/nameurlhttp://spark.mk.com/urlpropertiesproject.build.sourceEncodingUTF-8/project.build.sourceEncodingmaven.compiler.source1.8/maven.compiler.sourcemaven.compiler.target1.8/maven.compiler.targetscala.version2.11.1/scala.versionspark.version2.4.4/spark.versionhadoop.version2.6.0/hadoop.version/propertiesdependencies!-- scala依赖--dependencygroupIdorg.scala-lang/groupIdartifactIdscala-library/artifactIdversion${scala.version}/version/dependency!-- spark依赖--dependencygroupIdorg.apache.spark/groupIdartifactIdspark-core_2.11/artifactIdversion${spark.version}/version/dependencydependencygroupIdorg.apache.spark/groupIdartifactIdspark-sql_2.11/artifactIdversion${spark.version}/version/dependencydependencygroupIdjunit/groupIdartifactIdjunit/artifactIdversion4.11/versionscopetest/scope/dependency/dependenciesbuildpluginManagementpluginspluginartifactIdmaven-clean-plugin/artifactIdversion3.1.0/version/pluginpluginartifactIdmaven-resources-plugin/artifactIdversion3.0.2/version/pluginpluginartifactIdmaven-compiler-plugin/artifactIdversion3.8.0/version/pluginpluginartifactIdmaven-surefire-plugin/artifactIdversion2.22.1/version/pluginpluginartifactIdmaven-jar-plugin/artifactIdversion3.0.2/version/plugin/plugins/pluginManagement/build
/project 三、编程代码
public class MultiTableJoinApp implements SparkConfInfo {public static void main(String[] args) {String factoryFilePath E:\\spark\\factory.txt;SparkSession sparkSession new MultiTableJoinApp().getSparkConf(MultiTableJoinApp);JavaPairRDDString, String addressFactoryRdd sparkSession.sparkContext().textFile(factoryFilePath, 4).toJavaRDD().flatMap(v - Arrays.asList(v.split(\n)).iterator()).mapToPair(v - {if(v.matches(\\sfactoryname\\saddressed\\s)){return null;}String[] data v.trim().split(\\s{2,});if (data.length ! 2) {return null;}return new Tuple2(data[1],data[0]);}).filter(v - v ! null);String addressFilePath E:\\spark\\address.txt;JavaPairRDDString, String addressNameRdd sparkSession.sparkContext().textFile(addressFilePath, 4).toJavaRDD().flatMap(v - Arrays.asList(v.split(\n)).iterator()).mapToPair(v - {if(v.matches(\\saddressID\\saddressname\\s)){return null;}String[] data v.trim().split(\\s{2,});if (data.length ! 2) {return null;}return new Tuple2(data[0],data[1]);}).filter(v - v ! null);JavaPairRDDString, Tuple2String, String joinRdd addressFactoryRdd.join(addressNameRdd);ListTuple2String, String childGrand joinRdd.mapToPair(v-new Tuple2(v._2._1, v._2._2)).sortByKey(true).collect();System.out.println(factoryname\t\taddressname);childGrand.forEach(v - System.out.println(v._1 \t\t v._2));sparkSession.stop();}
}public interface SparkConfInfo {default SparkSession getSparkConf(String appName){SparkConf sparkConf new SparkConf();if(System.getProperty(os.name).toLowerCase().contains(win)) {sparkConf.setMaster(local[4]);System.out.println(使用本地模拟是spark);}else{sparkConf.setMaster(spark://hadoop01:7077,hadoop02:7077,hadoop03:7077);sparkConf.set(spark.driver.host,192.168.150.1);//本地ip必须与spark集群能够相互访问如同一个局域网sparkConf.setJars(new String[] {.\\out\\artifacts\\spark_test\\spark-test.jar});//项目构建生成的路径}SparkSession session SparkSession.builder().appName(appName).config(sparkConf).config(sparkConf).getOrCreate();return session;}
}factory.txt文件内容
factoryname addressed
Beijing Red Star 1
Shenzhen Thunder 3
Guangzhou Honda 2
Beijing Rising 1
Guangzhou Development Bank 2
Tencent 3
Back of Beijing 1
address.txt文件内容
addressID addressname
1 Beijing
2 Guangzhou
3 Shenzhen
4 Xian
输出
factoryname addressname
Back of Beijing Beijing
Beijing Red Star Beijing
Beijing Rising Beijing
Guangzhou Development Bank Guangzhou
Guangzhou Honda Guangzhou
Shenzhen Thunder Shenzhen
Tencent Shenzhen 四、join方法
W JavaPairRDDK, Tuple2V, W join(JavaPairRDDK, W other)
关联表返回相同可以的键值对
