Hive学习——自定义函数UDFUDTF

目录

一、添加依赖

二、编写自定义UDF函数

(一)自定义首字母大写函数

1.java代码

2.hive中运行

(二)自定义字符串全部小写的函数

1.java代码

2.hive运行

(三)创建解析JSON字符串的函数

1.java代码

三、自定义编写UDTF函数

1.java编写

2.hive运行

---

        虽然Hive中内置了很多函数，但是Hive也给我们提供了自定义函数的接口，方便我们自定义函数进行调用，从而减少代码的编写量。

一、添加依赖

<dependencies><dependency><groupId>org.apache.hive</groupId><artifactId>hive-exec</artifactId><version>3.1.2</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>3.1.3</version></dependency><dependency><groupId>org.apache.hive</groupId><artifactId>hive-jdbc</artifactId><version>3.1.2</version><exclusions><exclusion><groupId>org.glassfish</groupId><artifactId>javax.el</artifactId></exclusion><exclusion><groupId>org.eclipse.jetty</groupId><artifactId>jetty</artifactId></exclusion></exclusions></dependency><dependency><groupId>junit</groupId><artifactId>junit</artifactId><version>4.12</version><scope>test</scope></dependency></dependencies>

二、编写自定义UDF函数

如果在hive中新建一个函数，断联后新函数就会丢失，所以需要在java中编写，打包后放入lib中。

(一)自定义首字母大写函数

1.java代码

import java.util.Locale;/*** 将传入的字符串，首字母变成大写字母*/
public class InitialString extends UDF {public static void main(String[] args) {InitialString ini = new InitialString();String hello = ini.evaluate("hello");System.out.println(hello);}public String evaluate(final String txt) {return txt.trim().substring(0, 1).toUpperCase(Locale.ROOT) + txt.substring(1);}
}

maven编译打包，放到hive的lib目录下

2.hive中运行

-- 加载jar包
hive (default)>add jar /opt/soft/hive312/lib/hivestu-1.0-SNAPSHOT.jar-- 创建函数
hive (default)> create temporary function myudf as 'test.udf.InitialString';-- 类的全路径-- 调用函数并输入参数
hive (default)> select myudf('hello');
Hellohive (default)> select myudf('hadoop');
Hadoophive (default)> select myudf('java');
Java

(二)自定义字符串全部小写的函数

1.java代码

import org.apache.hadoop.hive.ql.exec.UDF;public class LowerUDF extends UDF {public static void main(String[] args) {LowerUDF lowerUDF = new LowerUDF();String evaluate = lowerUDF.evaluate("HELLO", "JAVA");System.out.println(evaluate);}public String evaluate(final String txt, final String txt2) {String res = txt + "," + txt2;return res.toLowerCase();}
}

2.hive运行

hive (default)> create function mylower as 'test.udf.LowerUDF';hive (default)> select mylower('HELLO','JAVA');
hello,java

(三)创建解析JSON字符串的函数

1.java代码

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.json.JSONObject;/*** "15850500365|{"name":"zhangsan","age":50,"address":"上海"}"*/
public class ThreeUDF extends UDF {public static void main(String[] args) {ThreeUDF threeUDF = new ThreeUDF();String tel = threeUDF.evaluate("15850500365|{\"name\":\"zhangsan\",\"age\":50,\"address\":\"上海\"}", "address");System.out.println(tel);}public String evaluate(String line, String key) {String[] infos = line.split("\\|");if (infos.length != 2 || StringUtils.isBlank(infos[1])) {return "";}if (key.equals("tel")) {return infos[0];} else {JSONObject object = new JSONObject(infos[1].trim());if (key.equals("name") && object.has("name"))return object.getString("name");else if (key.equals("age") && object.has("age"))return object.getString("age");else if (key.equals("address") && object.has("address"))return object.getString("address");}return "nothave";}
}

三、自定义编写UDTF函数

1.java编写

import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;import java.util.ArrayList;
import java.util.List;/*** UDTF解决输入一行，输出多行的需求* 输入"hello,world,zhangsan,shanghai",","* 输出* world* hello* world* zhangsan* shanghai*/
public class MyUDTF extends GenericUDTF {private List<String> wordList = new ArrayList<String>();@Overridepublic StructObjectInspector initialize(StructObjectInspector argOIs)throws UDFArgumentException {/*** 输出数据类型说明：*/List<String> fieldNames = new ArrayList<String>();fieldNames.add("word");List<ObjectInspector> fieldOIs = new ArrayList<>();fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);}@Overridepublic void process(Object[] args) throws HiveException {String data = args[0].toString();String splitkey = args[1].toString();String[] words = data.split(splitkey);for (String word :words) {wordList.clear();wordList.add(word);forward(wordList);}}@Overridepublic void close() throws HiveException {}
}

2.hive运行

hive (default)> create function myudtf as 'test.udtf.MyUDTF';hive (default)> select myudtf('aa,bb,cc,dd',',');
word
aa
bb
cc
ddhive (default)> select myudtf('aa$bb$cc$dd','$');
word
aa$bb$cc$ddhive (default)> select myudtf('aa$bb$cc$dd','\\$');
word
aa
bb
cc
dd