/
kythe.sh
157 lines (135 loc) · 4.12 KB
/
kythe.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
#!/bin/bash -e
# Copyright 2014 The Kythe Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o pipefail
export SHELL=/bin/bash
usage() {
cat >&2 <<EOF
usage: kythe [--repo git-url] [--extract extractor] [--index]
[--ignore-unhandled] [--files config-path] [--files-excludes re1,re2]
example: docker run --rm -t -v "$HOME/repo:/repo" -v "$HOME/gs:/graphstore" \
google/kythe --extract maven --index --files --files-excludes '(^|/)\.,^third_party'
Extraction:
If given an --extract type, the compilations in the mounted /repo VOLUME (or the given --repo
which will copied to /repo) will be extracted to the /compilations VOLUME w/ subdirectories for
each compilation's language (e.g. /compilations/java, /compilations/go).
Supported Extractors: maven
Indexing:
If given the --index flag, each compilation in /compilations will be sent to a corresponding
language indexer and the outputs will be stored in a GraphStore in the /graphstore VOLUME. If a
compilation is without a corresponding language indexer, an error will be reported unless
--ignore-unhandled is set.
To emit file nodes for the entire repository, use the --files flag to specify a JSON file VNames
configuration relative to the repository root. --files-excludes can be used to exclude certain
paths by a comma-separated list regex patterns. It is highly recommended to exclude build
output directories such as '(^|/)target'. The --index flag is required for --files to be handled.
Supported Languages: java,c++
EOF
}
usage_error() {
echo "ERROR: $*" >&2
usage
exit 1
}
error() {
echo "ERROR: $*" >&2
exit 1
}
cleanup() {
fix_permissions /repo
fix_permissions /compilations
fix_permissions /graphstore
fix_permissions /root/.m2
}
trap cleanup EXIT
REPO=
IGNORE_UNHANDLED=
EXTRACTOR=
INDEXING=
FILES_CONFIG=
FILES_EXCLUDES='(^|/)\.'
while [[ $# -gt 0 ]]; do
case "$1" in
--repo|-r)
REPO="$2"
shift ;;
--extract|-e)
EXTRACTOR="$2"
shift ;;
--files|-f)
FILES_CONFIG="$2"
shift ;;
--files-excludes)
FILES_EXCLUDES="$2"
shift ;;
--index|-i)
INDEXING=1 ;;
--ignore-unhandled)
IGNORE_UNHANDLED=1 ;;
--help|-h)
usage
exit 0 ;;
*) usage_error "Unknown argument: $1" ;;
esac
shift
done
mkdir -p /repo /compilations /graphstore
if [[ -n "$REPO" ]]; then
if [ ! "$(ls -A /repo)" ]; then
error '/repo not empty when given --repo'
fi
git clone "$REPO" /repo
fi
case "$EXTRACTOR" in
maven)
echo 'Extracting compilations' >&2
"${EXTRACTOR}_extractor" ;;
"")
echo 'Skipping extraction' >&2 ;;
*)
error "Unknown extractor: '$EXTRACTOR'" ;;
esac
if [[ -z "$INDEXING" ]]; then
echo 'Skipping indexing' >&2
exit
fi
drive_indexer_kzip() {
local lang
lang="$(basename "$(dirname "$1")")"
local analyzer="/kythe/bin/${lang}_indexer"
if [[ ! -x "$analyzer" ]]; then
if [[ -n "$IGNORE_UNHANDLED" ]]; then
return 0
else
echo "Unhandled index file for '$lang': $*" >&2
return 1
fi
fi
echo "Indexing $*" >&2
"$analyzer" "$@"
}
export -f drive_indexer_kzip
export IGNORE_UNHANDLED
find /compilations -name '*.kzip' | sort -R | \
{ parallel --gnu -L1 drive_indexer_kzip || echo "$? analysis failures" >&2; } | \
dedup_stream | \
write_entries --workers 12 --graphstore /graphstore
if [[ -z "$FILES_CONFIG" ]]; then
echo "Skipping repository files indexing" >&2
exit
fi
echo 'Emitting nodes for repository' >&2
cd /repo/ && \
index_repository --vnames "$FILES_CONFIG" --exclude "$FILES_EXCLUDES" | \
write_entries --workers 4 --graphstore /graphstore