Spark笔记1--以表头创建空的dataset

最近上手Spark相关的项目，实践过程中遇到的几个问题，来此记录下。

1.引入maven包

<dependency>

	<groupId>org.apache.spark</groupId>

	<artifactId>spark-core_2.11</artifactId>

	<version>2.3.0</version>

</dependency>



<dependency>

	<groupId>org.apache.spark</groupId>

	<artifactId>spark-sql_2.11</artifactId>

	<version>2.3.0</version>

</dependency>

2.给定一串表头，建立空的dataFrame

       public static void main(String[] args) {

		SparkSession sparkSession = SparkSession.builder().appName("Test").master("local")

               .config("spark.sql.inMemoryColumnarStorage.compressed", "true").getOrCreate();

		//给定一串表头

		String colstr = "编号,姓名,性别,年龄";

		//以,分割

		String[] cols = colstr.split(",");

		List<StructField> fields = new ArrayList<>();

		for (String fieldName : cols) {

			//创建StructField，因不知其类型，默认转为字符型

			StructField field = DataTypes.createStructField(fieldName, DataTypes.StringType, true);

			fields.add(field);

		}

		//创建StructType

		StructType schema = DataTypes.createStructType(fields);

		List<Row> rows = new ArrayList<>();

		//创建只包含schema的Dataset

		Dataset<Row> data = sparkSession.createDataFrame(rows, schema);

		data.show();

	}