【NLP】11⼤Java开源中⽂分词器的使⽤⽅法和分词效果对⽐
本⽂的⽬标有两个：
1、学会使⽤11⼤Java开源中⽂分词器
2、对⽐分析11⼤Java开源中⽂分词器的分词效果
本⽂给出了11⼤Java开源中⽂分词的使⽤⽅法以及分词结果对⽐代码，⾄于效果哪个好，那要⽤的⼈结合⾃⼰的应⽤场景⾃⼰来判断。
11⼤Java开源中⽂分词器，不同的分词器有不同的⽤法，定义的接⼝也不⼀样，我们先定义⼀个统⼀的接⼝：
Java代码
1. /**
2. * 获取⽂本的所有分词结果, 对⽐不同分词器结果
3. * @author 杨尚川
4. */
5. public interface WordSegmenter {
6. /**
7. * 获取⽂本的所有分词结果
8. * @param text ⽂本
9. * @return 所有的分词结果，去除重复
try catch的使用方法10. */
11. default public Set<String> seg(String text) {
12. return segMore(text).values().stream().Set());
13. }
14. /**
15. * 获取⽂本的所有分词结果
16. * @param text ⽂本
17. * @return 所有的分词结果，KEY 为分词器模式，VALUE 为分词器结果
18. */
19. public Map<String, String> segMore(String text);
20. }
从上⾯的定义我们知道，在Java中，同样的⽅法名称和参数，但是返回值不同，这种情况不可以使⽤重载。
这两个⽅法的区别在于返回值，每⼀个分词器都可能有多种分词模式，每种模式的分词结果都可能不相同，第⼀个⽅法忽略分词器模式，返回所有模式的所有不重复分词结果，第⼆个⽅法返回每⼀种分词器模式及其对应的分词结果。
在这⾥，需要注意的是我们使⽤了Java8中的新特性默认⽅法，并使⽤stream把⼀个map的value转换为不重复的集合。
下⾯我们利⽤这11⼤分词器来实现这个接⼝：
1、word分词器
Java代码
1. @Override
2. public Map<String, String> segMore(String text) {
3. Map<String, String> map = new HashMap<>();
4. for(SegmentationAlgorithm segmentationAlgorithm : SegmentationAlgorithm.values()){
5. map.Des(), seg(text, segmentationAlgorithm));
6. }
7. return map;
8. }
9. private static String seg(String text, SegmentationAlgorithm segmentationAlgorithm) {
10. StringBuilder result = new StringBuilder();
11. for(Word word : WordSegmenter.segWithStopWords(text, segmentationAlgorithm)){
12. result.Text()).append(" ");
13. }
14. String();
15. }
2、Ansj分词器
Java代码
1. @Override
2. public Map<String, String> segMore(String text) {
3. Map<String, String> map = new HashMap<>();
4.
5. StringBuilder result = new StringBuilder();
6. for(Term term : BaseAnalysis.parse(text)){
7. result.Name()).append(" ");
8. }
9. map.put("BaseAnalysis", String());
10.
11. result.setLength(0);
12. for(Term term : ToAnalysis.parse(text)){
13. result.Name()).append(" ");
14. }
15. map.put("ToAnalysis", String());
16.
17. result.setLength(0);
18. for(Term term : NlpAnalysis.parse(text)){
19. result.Name()).append(" ");
20. }
21. map.put("NlpAnalysis", String());
22.
23. result.setLength(0);
24. for(Term term : IndexAnalysis.parse(text)){
25. result.Name()).append(" ");
26. }
27. map.put("IndexAnalysis", String());
28.
29. return map;
30. }
3、Stanford分词器
Java代码
1. private static final StanfordCoreNLP CTB = new StanfordCoreNLP("StanfordCoreNLP-chinese-ctb");
2. private static final StanfordCoreNLP PKU = new StanfordCoreNLP("StanfordCoreNLP-chinese-pku");
3. private static final PrintStream NULL_PRINT_STREAM = new PrintStream(new NullOutputStream(), false);
4. p ublic Map<String, String> segMore(String text) {
5. Map<String, String> map = new HashMap<>();
6. map.put("Stanford Beijing University segmentation", seg(PKU, text));
7. map.put("Stanford Chinese Treebank segmentation", seg(CTB, text));
8. return map;
9. }
10. private static String seg(StanfordCoreNLP stanfordCoreNLP, String text){
11. PrintStream err = ;
12. System.setErr(NULL_PRINT_STREAM);
13. Annotation document = new Annotation(text);
14. stanfordCoreNLP.annotate(document);
15. List<CoreMap> sentences = (CoreAnnotations.SentencesAnnotation.class);
16. StringBuilder result = new StringBuilder();
17. for(CoreMap sentence: sentences) {
18. for (CoreLabel token: (CoreAnnotations.TokensAnnotation.class)) {
19. String word = (CoreAnnotations.TextAnnotation.class);;
20. result.append(word).append(" ");
21. }
22. }
23. System.setErr(err);
24. String();
25. }
4、FudanNLP分词器
Java代码
1. private static CWSTagger tagger = null;
2. static{
3. try{
4. tagger = new CWSTagger("lib/fudannlp_seg.m");
5. tagger.setEnFilter(true);
6. }catch(Exception e){
7. e.printStackTrace();
8. }
9. }
10. @Override
11. public Map<String, String> segMore(String text) {
12. Map<String, String> map = new HashMap<>();
13. map.put("FudanNLP", tagger.tag(text));
14. return map;
15. }
5、Jieba分词器
Java代码
1. private static final JiebaSegmenter JIEBA_SEGMENTER = new JiebaSegmenter();
2. @Override
3. public Map<String, String> segMore(String text) {
4. Map<String, String> map = new HashMap<>();
5. map.put("INDEX", seg(text, SegMode.INDEX));
6. map.put("SEARCH", seg(text, SegMode.SEARCH));
7. return map;
8. }
9. private static String seg(String text, SegMode segMode) {
10. StringBuilder result = new StringBuilder();
11. for(SegToken token : JIEBA_SEGMENTER.process(text, segMode)){
12. result.append(Token()).append(" ");
13. }
14. String();
15. }
6、Jcseg分词器
Java代码
1. private static final JcsegTaskConfig CONFIG = new JcsegTaskConfig();
2. private static final ADictionary DIC = ateDefaultDictionary(CONFIG);
3. static {
4. CONFIG.setLoadCJKSyn(false);
5. CONFIG.setLoadCJKPinyin(false);
6. }
7. @Override
8. public Map<String, String> segMore(String text) {
9. Map<String, String> map = new HashMap<>();
11. map.put("复杂模式", segText(text, JcsegTaskConfig.COMPLEX_MODE));
12. map.put("简易模式", segText(text, JcsegTaskConfig.SIMPLE_MODE));
13.
14. return map;
15. }
16. private String segText(String text, int segMode) {
17. StringBuilder result = new StringBuilder();
18. try {
19. ISegment seg = ateJcseg(segMode, new Object[]{new StringReader(text), CONFIG, DIC});
20. IWord word = null;
21. while((())!=null) {
22. result.Value()).append(" ");
23. }
24. } catch (Exception ex) {
25. throw new RuntimeException(ex);
26. }
27. String();
28. }
7、MMSeg4j分词器
Java代码
1. private static final Dictionary DIC = Instance();
2. private static final SimpleSeg SIMPLE_SEG = new SimpleSeg(DIC);
3. private static final ComplexSeg COMPLEX_SEG = new ComplexSeg(DIC);
4. private static final MaxWordSeg MAX_WORD_SEG = new MaxWordSeg(DIC);
5. @Override
6. public Map<String, String> segMore(String text) {
7. Map<String, String> map = new HashMap<>();
8. map.put(Class().getSimpleName(), segText(text, SIMPLE_SEG));
9. map.put(Class().getSimpleName(), segText(text, COMPLEX_SEG));
10. map.put(MAX_Class().getSimpleName(), segText(text, MAX_WORD_SEG));
11. return map;
12. }
13. private String segText(String text, Seg seg) {
14. StringBuilder result = new StringBuilder();
15. MMSeg mmSeg = new MMSeg(new StringReader(text), seg);
16. try {
17. Word word = null;
18. while((())!=null) {
19. result.String()).append(" ");
20. }
21. } catch (IOException ex) {
22. throw new RuntimeException(ex);
23. }
24. String();
25. }
8、IKAnalyzer分词器
Java代码
1. @Override
2. public Map<String, String> segMore(String text) {
3. Map<String, String> map = new HashMap<>();
4.
5. map.put("智能切分", segText(text, true));
6. map.put("细粒度切分", segText(text, false));
8. return map;
9. }
10. private String segText(String text, boolean useSmart) {
11. StringBuilder result = new StringBuilder();
12. IKSegmenter ik = new IKSegmenter(new StringReader(text), useSmart);
13. try {
14. Lexeme word = null;
15. while((())!=null) {
16. result.LexemeText()).append(" ");
17. }
18. } catch (IOException ex) {
19. throw new RuntimeException(ex);
20. }
21. String();
22. }
9、Paoding分词器
Java代码
1. private static final PaodingAnalyzer ANALYZER = new PaodingAnalyzer();
2. @Override
3. public Map<String, String> segMore(String text) {
4. Map<String, String> map = new HashMap<>();
5.
6. map.put("MOST_WORDS_MODE", seg(text, PaodingAnalyzer.MOST_WORDS_MODE));
7. map.put("MAX_WORD_LENGTH_MODE", seg(text, PaodingAnalyzer.MAX_WORD_LENGTH_MODE));
8.
9. return map;
10. }
11. private static String seg(String text, int mode){
12. ANALYZER.setMode(mode);
13. StringBuilder result = new StringBuilder();
14. try {
15. Token reusableToken = new Token();
16. TokenStream stream = kenStream("", new StringReader(text));
17. Token token = null;
18. while((token = (reusableToken)) != null){
19. result.()).append(" ");
20. }
21. } catch (Exception ex) {
22. throw new RuntimeException(ex);
23. }
24. String();
25. }
10、smartcn分词器
Java代码
1. p rivate static final SmartChineseAnalyzer SMART_CHINESE_ANALYZER = new SmartChineseAnalyzer();
2. @Override
3. public Map<String, String> segMore(String text) {
4. Map<String, String> map = new HashMap<>();
5. map.put("smartcn", segText(text));
6. return map;
7. }
8. private static String segText(String text) {
9. StringBuilder result = new StringBuilder();
版权声明：本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系QQ:729038198，我们将在24小时内删除。
发表评论