Skip to content

Commit

Permalink
improve dx
Browse files Browse the repository at this point in the history
  • Loading branch information
@felixdorn
felixdorn committed Nov 2, 2021
1 parent 6475f0d commit 0541115
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 68 deletions.
36 changes: 15 additions & 21 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,24 @@ use Felix\Tin\Tin;
$theme = new JetbrainsDark();
$tin = new Tin($theme);

echo $tin->process("<?php\n\necho 'Hello world';\n", ansi: true);
echo $tin->highlight("<?php\n\necho 'Hello world';\n");
```

You can disable the ansi output by passing `false` as the second parameter.

## Customizing the output

Apart from using a custom theme to change the colors, you have complete control over the highlighting proccess.

```php
use Felix\Tin\Token;

$tin->process(
$code,
fn(Token $token, Token $lastToken) => $token->line . '| ' . $token->repr()
)
```

## Themes

* [`Felix\Tin\Themes\JetbrainsDark`](src/Themes/JetbrainsDark.php)
Expand Down Expand Up @@ -68,35 +81,16 @@ class OneDark extends Theme
}
```

## Performance

The code has been optimized a lot as i needed to highlight files quickly. Therefore, some compromise were made in terms
of code readability and simplicity.

It takes on average 0.0007 second per file.

To put that in context, highlighting the whole PHPUnit library takes ~265ms.

Highlighting the vendor directory of this package takes ~1.78s. That's 1320 files per seconds.

> PHP built-in tokenizer for PHP uses most of the memory (around 80-90%)
You can check the full profiles here:

* [Highlighting PHPUnit](https://blackfire.io/profiles/2bd4c150-5226-4645-85fa-ffed43dc4602/graph)
* [Highlighting Vendor](https://blackfire.io/profiles/fa9b900f-d398-4efa-b999-9e7470b714b4/graph)

## Future

* PHPDoc
* Various outputs (cli / web)
* Line prefixes aka support for line numbers
* grayscale theme

## Known Issues

Named parameters are simply ignored by the built-in PHP parse which means that if you're named parameter is also a
keyword such as for, default. The highlighter wont pick up on it and will highlight it as a keyword rather than a named
keyword such as for, default. The highlighter won't pick up on it and will highlight it as a keyword rather than a named
parameter.

There is no solution to that problem unless we implement our own parser (no) or the parser gets fixed,
Expand Down
1 change: 1 addition & 0 deletions composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"php": "^8.0"
},
"require-dev": {
"blackfire/php-sdk": "^1.27",
"friendsofphp/php-cs-fixer": "^v2.19.2",
"pestphp/pest": "^v1.20.0",
"phpstan/phpstan": "^1.0.0",
Expand Down
94 changes: 56 additions & 38 deletions src/Tin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
namespace Felix\Tin;

use Felix\Tin\Themes\Theme;
use PhpToken;

class Tin
{
Expand All @@ -14,40 +13,44 @@ public function __construct(Theme $theme)
$this->theme = $theme;
}

public function process(string $code, bool $ansi = true): string
public static function from(Theme $theme): self
{
if (!$ansi) {
return $code;
}
return new self($theme);
}

$highlighted = '';
$tokens = PhpToken::tokenize($code);
public function highlight(string $code): string
{
return $this->process($code, function (Token $token): string {
return $token->text;
});
}

public function process(string $code, callable $transformer): string
{
$highlighted = '';
$lastToken = $tokens[array_key_last($tokens)];
$tokens = Token::tokenize($code);
$attributeOpen = false;
foreach ($tokens as $index => $token) {
if ($token->id !== T_STRING) {
if ($token->text === ':' && $tokens[$index - 1]->id === T_STRING) {
$id = T_NAMED_PARAMETER;
if ($token->text === ':' && $tokens[$index - 1]->id === T_NAMED_PARAMETER) {
$token->id = T_NAMED_PARAMETER;
} elseif ($attributeOpen && $token->text === ']' && ($tokens[$index + 1]->id === T_WHITESPACE || $tokens[$index + 1]->id === T_ATTRIBUTE)) {
$id = T_ATTRIBUTE_END;
$token->id = T_ATTRIBUTE_END;
$attributeOpen = false;
} else {
$id = $token->id;
} elseif ($token->id === T_ATTRIBUTE) {
$attributeOpen = true;
}
} elseif ($token->is(['true', 'false', 'null', 'string', 'int', 'float', 'object', 'callable', 'array', 'iterable', 'bool', 'self'])) {
$id = T_BUILTIN_TYPE;
$token->id = T_BUILTIN_TYPE;
} else {
$id = $this->idFromContext($tokens, $index);
}

if ($id === T_ATTRIBUTE) {
$attributeOpen = true;
$token->id = $this->idFromContext($tokens, $index);
}

if ($id < 256) {
if ($token->id < 256) {
$color = $this->theme->default;
} else {
$color = match ($id) {
$color = match ($token->id) {
T_METHOD_NAME, T_FUNCTION_DECL => $this->theme->function,
T_COMMENT, T_DOC_COMMENT => $this->theme->comment,
T_CONSTANT_ENCAPSED_STRING, T_ENCAPSED_AND_WHITESPACE => $this->theme->string,
Expand All @@ -60,7 +63,11 @@ public function process(string $code, bool $ansi = true): string
default => $this->theme->default
};
}
$highlighted .= "\e[38;2;" . $color . 'm' . $token->text . "\e[0m";

$text = $token->text;
$token->text = "\e[38;2;" . $color . 'm' . $token->text . "\e[0m";
$highlighted .= $transformer($token, $lastToken);
$token->text = $text;
}

return $highlighted;
Expand All @@ -76,12 +83,11 @@ public function process(string $code, bool $ansi = true): string
* - T_VARIABLE
* - T_DECLARE_PARAMETER
*
* @param array<int, PhpToken> $tokens
* @param array<int, Token> $tokens
*/
protected function idFromContext(array $tokens, int $index): int
{
$ahead = $this->read($tokens, $index + 1);

$ahead = $this->lookAhead($tokens, $index + 1);
if ($ahead->id === T_DOUBLE_COLON) {
return T_CLASS_NAME;
}
Expand All @@ -90,21 +96,17 @@ protected function idFromContext(array $tokens, int $index): int
return T_NAMED_PARAMETER;
}

$behind = $this->read($tokens, $index - 1, ltr: false);
$twoBehind = $this->read($tokens, $index - 2, ltr: false);
$behind = $this->lookBehind($tokens, $index - 1);
$twoBehind = $this->lookBehind($tokens, $index - 2);

return match ($behind->id) {
T_NEW, T_USE, T_PRIVATE, T_PROTECTED, T_PUBLIC, T_NAMESPACE, T_CLASS, T_INTERFACE, T_TRAIT, T_EXTENDS, T_IMPLEMENTS, T_INSTEADOF => T_CLASS_NAME,
T_NEW, T_USE, T_PRIVATE, T_PROTECTED, T_PUBLIC, T_NAMESPACE, T_CLASS, T_INTERFACE, T_TRAIT, T_EXTENDS, T_IMPLEMENTS, T_INSTEADOF, 58, 124, 63, 44 => T_CLASS_NAME,
T_FUNCTION => $twoBehind->is([T_PRIVATE, T_PROTECTED, T_PUBLIC, T_STATIC]) ? T_METHOD_NAME : T_FUNCTION_DECL,
T_OBJECT_OPERATOR => $ahead->text === '(' ? T_METHOD_NAME : T_VARIABLE,
T_DOUBLE_COLON => $ahead->text === '(' || $ahead->id === T_INSTEADOF || $ahead->id === T_AS ? T_METHOD_NAME : T_CONST_NAME,
T_AS => T_METHOD_NAME,
T_ATTRIBUTE => T_ATTRIBUTE_CLASS,
default => (function () use ($behind, $ahead, $twoBehind, $tokens, $index) {
if ($behind->is([':', '|', '?', ','])) {
return T_CLASS_NAME;
}

default => (function () use ($ahead, $twoBehind, $tokens, $index) {
if ($ahead->text === '(') {
return T_FUNCTION_NAME;
}
Expand All @@ -115,8 +117,8 @@ protected function idFromContext(array $tokens, int $index): int

// public function foo(Bar $bar)
// 6 54 32 0
// This is where the -4 comes from
if ($this->read($tokens, $index - 4, ltr: false)->is(T_FUNCTION)) {
// This is where the $index - 4 comes from
if ($this->lookBehind($tokens, $index - 4)->is(T_FUNCTION)) {
return T_CLASS_NAME;
}

Expand All @@ -127,16 +129,32 @@ protected function idFromContext(array $tokens, int $index): int
}

/**
* Finds the nearest non-whitespace token at a given index in a given direction (either ltr or rtl).
* Finds the nearest non-whitespace token at a given index from left to right.
*
* @param array<int, Token> $tokens
*/
protected function lookAhead(array $tokens, int $index): Token
{
$token = $tokens[$index];

if ($token->id === T_WHITESPACE) {
$token = $tokens[$index + 1];
}

return $token;
}

/**
* Finds the nearest non-whitespace token at a given index from right to left.
*
* @param array<int, PhpToken> $tokens
* @param array<int, Token> $tokens
*/
protected function read(array $tokens, int $index, bool $ltr = true): PhpToken
protected function lookBehind(array $tokens, int $index): Token
{
$token = $tokens[$index];

if ($token->id === T_WHITESPACE) {
$token = $tokens[$index + ($ltr ? 1 : -1)];
$token = $tokens[$index - 1];
}

return $token;
Expand Down
27 changes: 27 additions & 0 deletions src/Token.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<?php

namespace Felix\Tin;

use PhpToken;

class Token extends PhpToken
{
public function getTokenName(): string
{
if ($this->id < 256) {
return chr($this->id);
}

return match ($this->id) {
T_CLASS_NAME => 'T_CLASS_NAME',
T_FUNCTION_NAME => 'T_FUNCTION_NAME',
T_CONST_NAME => 'T_CONST_NAME',
T_BUILTIN_TYPE => 'T_BUILTIN_TYPE',
T_METHOD_NAME => 'T_METHOD_NAME',
T_FUNCTION_DECL => 'T_FUNCTION_DECL',
T_DECLARE_PARAMETER => 'T_DECLARE_PARAMETER',
T_ATTRIBUTE_CLASS => 'T_ATTRIBUTE_CLASS',
default => parent::getTokenName()
};
}
}
20 changes: 11 additions & 9 deletions tests/TinTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,28 @@

use Felix\Tin\Themes\OneDark;
use Felix\Tin\Tin;
use Felix\Tin\Token;
use PHPUnit\Framework\TestCase;
use function Spatie\Snapshots\assertMatchesTextSnapshot;

uses(TestCase::class);

it('can highlight', function () {
$Tin = new Tin(
new OneDark()
);
$tin = Tin::from(new OneDark());

$hl = $Tin->process(
file_get_contents(__DIR__ . '/fixtures/sample') ?: throw new RuntimeException('can not read sample file'),
$hl = $tin->highlight(
file_get_contents(__DIR__ . '/fixtures/sample'),
);

assertMatchesTextSnapshot($hl);
});

it('does not highlight with ANSI disabled', function () {
$code = file_get_contents(__DIR__ . '/fixtures/sample');
$hl = new Felix\Tin\Tin(new OneDark());
it('can customize highlighting output', function () {
$tin = Tin::from(new OneDark());

$output = $tin->process('<?php echo "Hello world";', function (Token $token, Token $lastToken) {
return $token->id;
});

expect($hl->process($code, false))->toBe($code);
expect($output)->toBe(T_OPEN_TAG . T_ECHO . T_WHITESPACE . T_CONSTANT_ENCAPSED_STRING . ord(';'));
});

0 comments on commit 0541115

Please sign in to comment.