pip install -U transformers torch

from transformers import BertTokenizer, BertModel
import torch

# Load pre-trained model and tokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

# Define the text
text = "The quick brown fox jumps over the lazy dog."
# Tokenize the text, 'pt' means PyTorch format
inputs = tokenizer(text, return_tensors='pt')

# Obtain the embeddings
with torch.no_grad():
    outputs = model(**inputs)
# Extract the last hidden state (embeddings)
last_hidden_states = outputs.last_hidden_state

# Print the dimensions of the embeddings
print("Shape of the last hidden state (embeddings):", 
      last_hidden_states.shape)

# Print embeddings for each token along with their vector dimension
tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])
for token, embedding in zip(tokens, last_hidden_states[0]):
    print(f"Token: {token}, Embedding Dimension: {embedding.shape}, 
          Embedding (first 5 components): {embedding[:5]}...")  
          # Display first 5 components for brevity

from transformers import BertTokenizer, BertModel
import torch

# Load pre-trained model and tokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

# Define the text
text = "The quick brown fox jumps over the lazy dog."

# Tokenize the text
inputs = tokenizer(text, return_tensors='pt')

# Obtain the embeddings
with torch.no_grad():
    outputs = model(**inputs)

# Extract the last hidden state (embeddings)
last_hidden_states = outputs.last_hidden_state

# Print the dimensions of the embeddings
print("Shape of the last hidden state (embeddings):", last_hidden_states.shape)

# Print embeddings for each token along with their vector dimension
tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])
for token, embedding in zip(tokens, last_hidden_states[0]):
    print(f"Token: {token}, Embedding Dimension: {embedding.shape}, Embedding (first 5 components): {embedding[:5]}...") 
    # Display first 5 components for brevity

''' this is not a module '''
import os
import pandas as pd
import tiktoken
from openai import OpenAI

client = OpenAI()  # OpenAI API client，須設定環境變數OPENAI_API_KEY

#
# 計算embedding vector
#
def get_embedding(text, model="text-embedding-3-small"):
    '''呼叫OpenAI API, 計算embedding vector
       text: 要計算embedding vector的字串
       model: embedding model, 預設值text-embedding-3-small, 其他選項參考OpenAI官網
    '''

    text = text.replace("\n", " ")    # 置換「換行字元」為空白字元
    return client.embeddings.create(input=[text], model=model).data[0].embedding

# 範例主程式
# 
# 1. 讀取資料
absolute_path = os.path.dirname(__file__)  # 此py檔案的絕對路徑
FILENAME = "專題.csv"              # 資料檔，共有10個欄位
file_path = os.path.join(absolute_path, FILENAME)  # 組成資料檔的絕對路徑

df = pd.read_csv(file_path, index_col=False)   # 讀取資料，建立data frame

df = df[["Q", "A"]] # 挑取Q, Al兩個欄位
df = df.dropna()    # 資料清理

print(df.head(2))   # 印出前2筆，看看是否正確

#df.drop("Time", axis=1, inplace=True)

EMBEDDING_MODEL = "text-embedding-3-small"
EMBEDDING_ENCODING = "cl100k_base"
MAX_TOKENS = 8000  # the maximum for text-embedding-3-small is 8191

encoding = tiktoken.get_encoding(EMBEDDING_ENCODING)

# omit reviews that are too long to embed
#df["n_tokens"] = df.combined.apply(lambda x: len(encoding.encode(x)))
df["n_tokens"] = df.Q.apply(lambda x: len(encoding.encode(x)))
df = df[df.n_tokens <= MAX_TOKENS]
print(len(df))

###  Get embeddings and save them for future reuse
df["embedding"] = df.Q.apply(lambda x: get_embedding(x, model=EMBEDDING_MODEL))
df.to_csv('review_short_embeddings.csv')

a = get_embedding("hi", model=EMBEDDING_MODEL)
print(f'embedding for "hi":{a}')

import os
import pandas as pd
from ast import literal_eval
import tiktoken
from openai import OpenAI
from scipy.spatial.distance import cosine

client = OpenAI()  # OpenAI API client，須設定環境變數OPENAI_API_KEY

#
# 計算embedding vector
#
def get_embedding(text, model="text-embedding-3-small"):
    '''呼叫OpenAI API, 計算embedding vector
       text: 要計算embedding vector的字串
       model: embedding model, 預設值text-embedding-3-small, 其他選項參考OpenAI官網
    '''

    text = text.replace("\n", " ")    # 置換「換行字元」為空白字元
    return client.embeddings.create(input=[text], model=model).data[0].embedding

#
# 讀取知識庫含embedding vector。
#
def get_knowledgebase_with_embedding(file_path):
    df = pd.read_csv(file_path, index_col=0) #讀取資料
    # print(df.head(2))
    return df

#
# 計算embedding vector之間的距離
#
def cosine_distance(v_a, v_b):
    value = cosine(v_a, v_b)
    return value

#--------------------------------------------    
# 範例主程式：
# 新進問題 與 知識庫中儲存問題相比較
# 逐一比對後，將embedding向量相近者對應之「答案」取出
#-------------------------------------------- 

# 0. 前置作業讀取資料與向量檔案
EMBEDDING_MODEL = "text-embedding-3-small"
filename = 'review_short_embeddings.csv'  # 範例檔欄位 ,Q,A,n_tokens,embedding
absolute_path = os.path.dirname(__file__)  # 此py檔案的絕對路徑
file_path = os.path.join(absolute_path, filename)  # 組成資料檔的絕對路徑
df = get_knowledgebase_with_embedding(file_path)   # 讀取檔案存入dataframe 

# 1. 使用者輸入問題後，將問題與知識庫的向量做比對
question = "如何取件?"  # 測試用的新進問題
v_question = get_embedding(question, model=EMBEDDING_MODEL)
# 比較v_question與 df中所有問題的向量
# 呼叫literal_eval()是因為當初寫出檔案時，已將浮點陣列[]變成字串'[]'，故此處再轉回來
df["distance"] = df.embedding.apply(lambda x: cosine_distance(literal_eval(x), v_question))

# 1.1排序
df2 = df.sort_values(by=['distance'], ascending=True)
print(df2.head(2))

# 2. 生成
# 2.1 OpenAI參數設定
model = "gpt-3.5-turbo"     # openai model
temperature = 0           # 回應變化的多樣性，0：變化最少, 1:變化最多
max_tokens = 200            # 最多200個tokens

# 2.2 問題設定(few-shot prompting)
num_of_shot = 3             # few-shot prompting(生成時參考的案例數)
instruction ="請依下列內容, 重整後產生答案。"
messages = [               
    {
        "role": "system",
        "content": instruction
    }
]           

candidate = df2.iloc[0:num_of_shot, lambda df2:[0,1]]   # 挑出Q,A兩欄, 前num_of_shot名，作為prompt候選內容
print(f'攫取的問答：{candidate.values.tolist()}')
# few-shot prompting, 依序加上前例
for item in candidate.values.tolist():
    prompt_u = {
        "role": "user",
        "content": item[0]
    }
    messages.append(prompt_u)
    prompt_a = {
        "role": 'assistant',
        "content": item[1]
    }
    messages.append(prompt_a) # 依序加在其後, 要留意max token of context的限制
# 2.3 加上問題
messages.append({
    "role": "user",
    "content": question
})

print('組成的訊息',messages)
# 2.4 生成
response = client.chat.completions.create(
    model = model,
    messages = messages,
    temperature= temperature,
    max_tokens=max_tokens
)
print(response.choices[0].message.content)

from ckip_transformers.nlp import CkipWordSegmenter, CkipPosTagger, CkipNerChunker
## 將斷詞結果words與 part-of-speech結果pos打包在一起
def pack_ws_pos_sentece(words, pos):
   assert len(words) == len(pos), f'words{len(words)}和pos{len(pos)}長度必須一樣'
   result = []   # 最終結果串列
   for word, p in zip(words, pos):  # zip將words,pos對應位置鏈在一起
      result.append(f"{word}({p})")
   return "\u3000".join(result)  #\u3000是全形空白字元

# Input text
text = [
   "華氏80度，氣壓30-6，整天下雨。前進到奇武荖，聆聽朗誦。然後珍珠里簡和冬瓜山，晚間在珍珠里簡的禮拜有一百五十人，拔了七十六顆牙。",
   "相當涼爽舒適，與阿華出去鄉間拜訪他的一位老朋友，是一位農人。我們被看見，認出來，且受排斥。遭到兩隻大黑狗攻擊，孩童吼叫，狂暴的辱罵。 我們回家，在吃過飯之後，我們比過去更勤奮的研讀各自的功課。",
   "華氏84度，陰天，氣壓30-10。上午七點離開大里簡，大部分都用走的，上午時點三十分來到頭城，拔了一些牙。十一點到打馬煙，我們剛到雨就來了。禮德醫師在路上幫一個傢伙縫手指。新漆好的禮拜堂到處都是用藍色和紅色漆，全部由他們自費花了十二元。禮德醫師拍團體照。我們照顧了十四名病人，還拔了一些牙。主持聖餐禮，聆聽十七個人背誦。大家都做得很好，我們發禮物給他們。"
   ]

ws_driver = CkipWordSegmenter(model="bert-base") # 載入 斷詞模型
pos_driver = CkipPosTagger(model="bert-base")    # 載入 POS模型
ner_driver = CkipNerChunker(model="bert-base")   # 載入 實體辨識模型
print('all loaded...')

# 執行pipeline 產生結果
ws = ws_driver(text)   # 斷詞
pos = pos_driver(ws)   # 詞性: 注意斷詞-詞性是依序完成
ner = ner_driver(text) # 實體辨識

##列印結果
for sentence, word, p, n in zip(text, ws, pos, ner):
    print(sentence)
    print(pack_ws_pos_sentece(word, p))
    for token in n:
        print(f"({token.idx[0]}, {token.idx[1]}, '{token.ner}', '{token.word}')")
    print()