upstream-archive/textgen
1
0
Fork 0
mirror of https://github.com/oobabooga/textgen.git synced 2026-07-23 11:20:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

Logits display improvements (#7486)

Browse Source
This commit is contained in:
wiger3
2026-04-15 03:29:19 +02:00
committed by GitHub
parent fbd95bd5e6
commit 75bf2feb59
2 changed files with 31 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
modules/llama_cpp_server.py
View File

@@ -296,19 +296,26 @@ class LlamaServer:
pprint.PrettyPrinter(indent=4, sort_dicts=False).pprint(printable_payload)
print()
for retry in range(5):
response = self.session.post(url, json=payload)
result = response.json()
def _try_fetch_logits():
for retry in range(5):
response = self.session.post(url, json=payload)
result = response.json()
if "completion_probabilities" in result:
if use_samplers:
return result["completion_probabilities"][0]["top_probs"]
else:
return result["completion_probabilities"][0]["top_logprobs"]
if "completion_probabilities" in result:
if use_samplers:
return result["completion_probabilities"][0]["top_probs"]
else:
return result["completion_probabilities"][0]["top_logprobs"]
time.sleep(0.05)
else:
raise Exception(f"Unexpected response format: 'completion_probabilities' not found in {result}")
time.sleep(0.05)
else:
raise Exception(f"Unexpected response format: 'completion_probabilities' not found in {result}")
result = _try_fetch_logits()
for entry in result:
if not entry.get('token'):
entry['token'] = self.decode([entry['id']])
return result
def get_prompt_logprob_entries(self, token_ids, n_probs=5, prompt=""):
"""Get logprob entries for prompt tokens via a single n_predict=0 request.

20
modules/logits.py
View File

@@ -36,15 +36,22 @@ def _get_next_logits(prompt, state, use_samplers, previous, top_logits=25, retur
return error_message, previous
# llama.cpp case
def _escaped(token):
chars = []
for a in token:
# C0 and DEL and C1
if ord(a) <= 0x1F or 0x7F <= ord(a) <= 0x9F:
chars.append(repr(a)[1:-1])
else:
chars.append(a)
return ''.join(chars)
if shared.model.__class__.__name__ == 'LlamaServer':
logprobs = shared.model.get_logits(prompt, state, n_probs=top_logits, use_samplers=use_samplers)
if return_dict:
output = {}
for entry in logprobs:
token = repr(entry['token'])
if len(token) > 2 and token.startswith("'") and token.endswith("'"):
token = token[1:-1]
token = _escaped(entry['token'])
prob = entry['prob'] if use_samplers else np.exp(entry['logprob'])
output[token] = prob
@@ -52,12 +59,11 @@ def _get_next_logits(prompt, state, use_samplers, previous, top_logits=25, retur
else:
output = ''
for entry in logprobs:
token = repr(entry['token'])
if len(token) > 2 and token.startswith("'") and token.endswith("'"):
token = token[1:-1]
token = _escaped(entry['token'])
token_id = entry['id']
prob = entry['prob'] if use_samplers else np.exp(entry['logprob'])
output += f"{prob:.5f} - {token}\n"
output += f"{prob:.5f} - [{token}] ({token_id})\n"
return output, previous
# All other model types