LLVM clang_tokenize 的示例

1，环境搭建

如同上篇

git clong llvm-project

git checkout llmvorg-3.4.0

# or llvmorg-18.1.rc

/bin/bash ./confiure --prefix=.......

# or cmake =Debug

make -j32

make -j install

2，示例代码

Makefile：如同上篇

LLVM_CONFIG ?= llvm-config
#CXX := clang++
ifndef VERBOSE
QUIET :=@
endifSRC_DIR ?= $(PWD)
LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags) 
COMMON_FLAGS = -Wall -Wextra
CXXFLAGS += $(COMMON_FLAGS) $(shell $(LLVM_CONFIG) --cxxflags)
LCXX :=$(shell $(LLVM_CONFIG) --cxxflags)
CPPFLAGS += $(shell $(LLVM_CONFIG) --cppflags) -I$(SRC_DIR)CLANGLIBS = \-Wl,--start-group \-lclang \-lclangFrontend \-lclangDriver \-lclangSerialization \-lclangParse \-lclangSema \-lclangAnalysis \-lclangEdit \-lclangAST \-lclangLex \-lclangBasic \-Wl,--end-groupLLVMLIBS = $(shell $(LLVM_CONFIG) --libs)PROJECT = token_analy
PROJECT_OBJECTS = token_analy.odefault: $(PROJECT)%.o : $(SRC_DIR)/%.cpp@echo Compiling $*.cpp$(QUIET)$(CXX) -c $(CPFLAGS) $(CXXFLAGS) $<$(PROJECT) : $(PROJECT_OBJECTS) @echo Linking $@$(QUIET)$(CXX) -o $@ $(LDFLAGS) $^ $(CLANGLIBS) $(LLVMLIBS) -lncurses.PHONY: clean
clean:$(QUIET)rm -f $(PROJECT) $(PROJECT_OBJECTS).PHONY: echo
echo:@echo "CXX 	is	$(CXX)"@echo "LDFLAGS 	is	$(LDFLAGS)}"@echo "CXXFLAGS	is	$(CXXFLAGS)"@echo "CPPFLAGS	is	$(CPPFLAGS)"@echo "SRC_DIR	is	$(SRC_DIR)"

token_analy.c:

extern "C"{
#include "clang-c/Index.h"
}#include "llvm/Support/CommandLine.h"
#include <iostream>using namespace llvm;static cl::opt<std::string> FileName(cl::Positional, cl::desc("Input file"), cl::Required);int main(int argc, char** argv)
{cl::ParseCommandLineOptions(argc, argv, "My tokenizer\n");CXIndex index = clang_createIndex(0, 0);const char *args[]={"-I/usr/include","-I."};CXTranslationUnit translationUnit = clang_parseTranslationUnit(index, FileName.c_str(), args, 2, NULL, 0, CXTranslationUnit_None);CXFile file = clang_getFile(translationUnit, FileName.c_str());CXSourceLocation loc_start = clang_getLocationForOffset(translationUnit, file, 0);CXSourceLocation loc_end   = clang_getLocationForOffset(translationUnit, file, 60);CXSourceRange range = clang_getRange(loc_start, loc_end);unsigned numTokens = 0;CXToken *tokens = NULL;clang_tokenize(translationUnit, range, &tokens, &numTokens);for(unsigned i=0; i<numTokens; ++i){enum CXTokenKind kind = clang_getTokenKind(tokens[i]);CXString name = clang_getTokenSpelling(translationUnit, tokens[i]);switch(kind){case CXToken_Punctuation:std::cout<<"PUNCTUATION("<<clang_getCString(name)<<") ";break;case CXToken_Keyword:std::cout<<"KEYWORD("<<clang_getCString(name)<<") ";break;case CXToken_Identifier:std::cout<<"IDENTIFIER("<<clang_getCString(name)<<") ";break;case CXToken_Literal:std::cout<<"COMMENT("<<clang_getCString(name)<<") ";break;default:std::cout<<"UNKNOWN("<<clang_getCString(name)<<") ";break;}clang_disposeString(name);}std::cout<< std::endl;clang_disposeTokens(translationUnit, tokens, numTokens);clang_disposeTranslationUnit(translationUnit);return 0;}

make